1 /* 2 * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others 3 * 4 * This program and the accompanying materials are made available under the 5 * terms of the Eclipse Distribution License v. 1.0 which is available at 6 * https://www.eclipse.org/org/documents/edl-v10.php. 7 * 8 * SPDX-License-Identifier: BSD-3-Clause 9 */ 10 package org.eclipse.jgit.lfs; 11 12 import static java.nio.charset.StandardCharsets.UTF_8; 13 import static org.eclipse.jgit.lfs.Protocol.OPERATION_UPLOAD; 14 import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest; 15 import static org.eclipse.jgit.transport.http.HttpConnection.HTTP_OK; 16 import static org.eclipse.jgit.util.HttpSupport.METHOD_POST; 17 import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT; 18 19 import java.io.IOException; 20 import java.io.InputStream; 21 import java.io.InputStreamReader; 22 import java.io.OutputStream; 23 import java.io.PrintStream; 24 import java.nio.file.Files; 25 import java.nio.file.Path; 26 import java.text.MessageFormat; 27 import java.util.Collection; 28 import java.util.HashMap; 29 import java.util.List; 30 import java.util.Map; 31 import java.util.Set; 32 import java.util.TreeSet; 33 34 import org.eclipse.jgit.api.errors.AbortedByHookException; 35 import org.eclipse.jgit.errors.IncorrectObjectTypeException; 36 import org.eclipse.jgit.errors.MissingObjectException; 37 import org.eclipse.jgit.hooks.PrePushHook; 38 import org.eclipse.jgit.lfs.Protocol.ObjectInfo; 39 import org.eclipse.jgit.lfs.errors.CorruptMediaFile; 40 import org.eclipse.jgit.lfs.internal.LfsConnectionFactory; 41 import org.eclipse.jgit.lfs.internal.LfsText; 42 import org.eclipse.jgit.lib.AnyObjectId; 43 import org.eclipse.jgit.lib.Constants; 44 import org.eclipse.jgit.lib.ObjectId; 45 import org.eclipse.jgit.lib.ObjectReader; 46 import org.eclipse.jgit.lib.Ref; 47 import org.eclipse.jgit.lib.RefDatabase; 48 import org.eclipse.jgit.lib.Repository; 49 import org.eclipse.jgit.revwalk.ObjectWalk; 50 import org.eclipse.jgit.revwalk.RevObject; 51 import org.eclipse.jgit.transport.RemoteRefUpdate; 52 import org.eclipse.jgit.transport.http.HttpConnection; 53 54 import com.google.gson.Gson; 55 import com.google.gson.stream.JsonReader; 56 57 /** 58 * Pre-push hook that handles uploading LFS artefacts. 59 * 60 * @since 4.11 61 */ 62 public class LfsPrePushHook extends PrePushHook { 63 64 private static final String EMPTY = ""; //$NON-NLS-1$ 65 private Collection<RemoteRefUpdate> refs; 66 67 /** 68 * @param repo 69 * the repository 70 * @param outputStream 71 * not used by this implementation 72 */ LfsPrePushHook(Repository repo, PrintStream outputStream)73 public LfsPrePushHook(Repository repo, PrintStream outputStream) { 74 super(repo, outputStream); 75 } 76 77 /** 78 * @param repo 79 * the repository 80 * @param outputStream 81 * not used by this implementation 82 * @param errorStream 83 * not used by this implementation 84 * @since 5.6 85 */ LfsPrePushHook(Repository repo, PrintStream outputStream, PrintStream errorStream)86 public LfsPrePushHook(Repository repo, PrintStream outputStream, 87 PrintStream errorStream) { 88 super(repo, outputStream, errorStream); 89 } 90 91 @Override setRefs(Collection<RemoteRefUpdate> toRefs)92 public void setRefs(Collection<RemoteRefUpdate> toRefs) { 93 this.refs = toRefs; 94 } 95 96 @Override call()97 public String call() throws IOException, AbortedByHookException { 98 Set<LfsPointer> toPush = findObjectsToPush(); 99 if (toPush.isEmpty()) { 100 return EMPTY; 101 } 102 HttpConnection api = LfsConnectionFactory.getLfsConnection( 103 getRepository(), METHOD_POST, OPERATION_UPLOAD); 104 Map<String, LfsPointer> oid2ptr = requestBatchUpload(api, toPush); 105 uploadContents(api, oid2ptr); 106 return EMPTY; 107 108 } 109 findObjectsToPush()110 private Set<LfsPointer> findObjectsToPush() throws IOException, 111 MissingObjectException, IncorrectObjectTypeException { 112 Set<LfsPointer> toPush = new TreeSet<>(); 113 114 try (ObjectWalk walk = new ObjectWalk(getRepository())) { 115 for (RemoteRefUpdate up : refs) { 116 walk.setRewriteParents(false); 117 excludeRemoteRefs(walk); 118 walk.markStart(walk.parseCommit(up.getNewObjectId())); 119 while (walk.next() != null) { 120 // walk all commits to populate objects 121 } 122 findLfsPointers(toPush, walk); 123 } 124 } 125 return toPush; 126 } 127 findLfsPointers(Set<LfsPointer> toPush, ObjectWalk walk)128 private static void findLfsPointers(Set<LfsPointer> toPush, ObjectWalk walk) 129 throws MissingObjectException, IncorrectObjectTypeException, 130 IOException { 131 RevObject obj; 132 ObjectReader r = walk.getObjectReader(); 133 while ((obj = walk.nextObject()) != null) { 134 if (obj.getType() == Constants.OBJ_BLOB 135 && getObjectSize(r, obj) < LfsPointer.SIZE_THRESHOLD) { 136 LfsPointer ptr = loadLfsPointer(r, obj); 137 if (ptr != null) { 138 toPush.add(ptr); 139 } 140 } 141 } 142 } 143 getObjectSize(ObjectReader r, RevObject obj)144 private static long getObjectSize(ObjectReader r, RevObject obj) 145 throws IOException { 146 return r.getObjectSize(obj.getId(), Constants.OBJ_BLOB); 147 } 148 loadLfsPointer(ObjectReader r, AnyObjectId obj)149 private static LfsPointer loadLfsPointer(ObjectReader r, AnyObjectId obj) 150 throws IOException { 151 try (InputStream is = r.open(obj, Constants.OBJ_BLOB).openStream()) { 152 return LfsPointer.parseLfsPointer(is); 153 } 154 } 155 excludeRemoteRefs(ObjectWalk walk)156 private void excludeRemoteRefs(ObjectWalk walk) throws IOException { 157 RefDatabase refDatabase = getRepository().getRefDatabase(); 158 List<Ref> remoteRefs = refDatabase.getRefsByPrefix(remote()); 159 for (Ref r : remoteRefs) { 160 ObjectId oid = r.getPeeledObjectId(); 161 if (oid == null) { 162 oid = r.getObjectId(); 163 } 164 if (oid == null) { 165 // ignore (e.g. symbolic, ...) 166 continue; 167 } 168 RevObject o = walk.parseAny(oid); 169 if (o.getType() == Constants.OBJ_COMMIT 170 || o.getType() == Constants.OBJ_TAG) { 171 walk.markUninteresting(o); 172 } 173 } 174 } 175 remote()176 private String remote() { 177 String remoteName = getRemoteName() == null 178 ? Constants.DEFAULT_REMOTE_NAME 179 : getRemoteName(); 180 return Constants.R_REMOTES + remoteName; 181 } 182 requestBatchUpload(HttpConnection api, Set<LfsPointer> toPush)183 private Map<String, LfsPointer> requestBatchUpload(HttpConnection api, 184 Set<LfsPointer> toPush) throws IOException { 185 LfsPointer[] res = toPush.toArray(new LfsPointer[0]); 186 Map<String, LfsPointer> oidStr2ptr = new HashMap<>(); 187 for (LfsPointer p : res) { 188 oidStr2ptr.put(p.getOid().name(), p); 189 } 190 Gson gson = Protocol.gson(); 191 api.getOutputStream().write( 192 gson.toJson(toRequest(OPERATION_UPLOAD, res)).getBytes(UTF_8)); 193 int responseCode = api.getResponseCode(); 194 if (responseCode != HTTP_OK) { 195 throw new IOException( 196 MessageFormat.format(LfsText.get().serverFailure, 197 api.getURL(), Integer.valueOf(responseCode))); 198 } 199 return oidStr2ptr; 200 } 201 uploadContents(HttpConnection api, Map<String, LfsPointer> oid2ptr)202 private void uploadContents(HttpConnection api, 203 Map<String, LfsPointer> oid2ptr) throws IOException { 204 try (JsonReader reader = new JsonReader( 205 new InputStreamReader(api.getInputStream(), UTF_8))) { 206 for (Protocol.ObjectInfo o : parseObjects(reader)) { 207 if (o.actions == null) { 208 continue; 209 } 210 LfsPointer ptr = oid2ptr.get(o.oid); 211 if (ptr == null) { 212 // received an object we didn't request 213 continue; 214 } 215 Protocol.Action uploadAction = o.actions.get(OPERATION_UPLOAD); 216 if (uploadAction == null || uploadAction.href == null) { 217 continue; 218 } 219 220 Lfs lfs = new Lfs(getRepository()); 221 Path path = lfs.getMediaFile(ptr.getOid()); 222 if (!Files.exists(path)) { 223 throw new IOException(MessageFormat 224 .format(LfsText.get().missingLocalObject, path)); 225 } 226 uploadFile(o, uploadAction, path); 227 } 228 } 229 } 230 parseObjects(JsonReader reader)231 private List<ObjectInfo> parseObjects(JsonReader reader) { 232 Gson gson = new Gson(); 233 Protocol.Response resp = gson.fromJson(reader, Protocol.Response.class); 234 return resp.objects; 235 } 236 uploadFile(Protocol.ObjectInfo o, Protocol.Action uploadAction, Path path)237 private void uploadFile(Protocol.ObjectInfo o, 238 Protocol.Action uploadAction, Path path) 239 throws IOException, CorruptMediaFile { 240 HttpConnection contentServer = LfsConnectionFactory 241 .getLfsContentConnection(getRepository(), uploadAction, 242 METHOD_PUT); 243 contentServer.setDoOutput(true); 244 try (OutputStream out = contentServer 245 .getOutputStream()) { 246 long size = Files.copy(path, out); 247 if (size != o.size) { 248 throw new CorruptMediaFile(path, o.size, size); 249 } 250 } 251 int responseCode = contentServer.getResponseCode(); 252 if (responseCode != HTTP_OK) { 253 throw new IOException(MessageFormat.format( 254 LfsText.get().serverFailure, contentServer.getURL(), 255 Integer.valueOf(responseCode))); 256 } 257 } 258 } 259