1 /* 2 * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others 3 * 4 * This program and the accompanying materials are made available under the 5 * terms of the Eclipse Distribution License v. 1.0 which is available at 6 * https://www.eclipse.org/org/documents/edl-v10.php. 7 * 8 * SPDX-License-Identifier: BSD-3-Clause 9 */ 10 package org.eclipse.jgit.lfs.internal; 11 12 import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; 13 import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT; 14 import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; 15 import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE; 16 17 import java.io.IOException; 18 import java.net.ProxySelector; 19 import java.net.URISyntaxException; 20 import java.net.URL; 21 import java.time.LocalDateTime; 22 import java.time.ZoneOffset; 23 import java.time.format.DateTimeFormatter; 24 import java.util.LinkedList; 25 import java.util.Map; 26 import java.util.TreeMap; 27 28 import org.eclipse.jgit.annotations.NonNull; 29 import org.eclipse.jgit.errors.CommandFailedException; 30 import org.eclipse.jgit.lfs.LfsPointer; 31 import org.eclipse.jgit.lfs.Protocol; 32 import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException; 33 import org.eclipse.jgit.lib.ConfigConstants; 34 import org.eclipse.jgit.lib.Repository; 35 import org.eclipse.jgit.lib.StoredConfig; 36 import org.eclipse.jgit.transport.HttpConfig; 37 import org.eclipse.jgit.transport.HttpTransport; 38 import org.eclipse.jgit.transport.URIish; 39 import org.eclipse.jgit.transport.http.HttpConnection; 40 import org.eclipse.jgit.util.HttpSupport; 41 import org.eclipse.jgit.util.SshSupport; 42 43 /** 44 * Provides means to get a valid LFS connection for a given repository. 45 */ 46 public class LfsConnectionFactory { 47 48 private static final int SSH_AUTH_TIMEOUT_SECONDS = 30; 49 private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$ 50 private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$ 51 private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>(); 52 53 /** 54 * Determine URL of LFS server by looking into config parameters lfs.url, 55 * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed 56 * from remote.[remote].url by appending "/info/lfs". In case there is no 57 * URL configured, a SSH remote URI can be used to auto-detect the LFS URI 58 * by using the remote "git-lfs-authenticate" command. 59 * 60 * @param db 61 * the repository to work with 62 * @param method 63 * the method (GET,PUT,...) of the request this connection will 64 * be used for 65 * @param purpose 66 * the action, e.g. Protocol.OPERATION_DOWNLOAD 67 * @return the url for the lfs server. e.g. 68 * "https://github.com/github/git-lfs.git/info/lfs" 69 * @throws IOException 70 */ getLfsConnection(Repository db, String method, String purpose)71 public static HttpConnection getLfsConnection(Repository db, String method, 72 String purpose) throws IOException { 73 StoredConfig config = db.getConfig(); 74 Map<String, String> additionalHeaders = new TreeMap<>(); 75 String lfsUrl = getLfsUrl(db, purpose, additionalHeaders); 76 URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT); 77 HttpConnection connection = HttpTransport.getConnectionFactory().create( 78 url, HttpSupport.proxyFor(ProxySelector.getDefault(), url)); 79 connection.setDoOutput(true); 80 if (url.getProtocol().equals(SCHEME_HTTPS) 81 && !config.getBoolean(HttpConfig.HTTP, 82 HttpConfig.SSL_VERIFY_KEY, true)) { 83 HttpSupport.disableSslVerify(connection); 84 } 85 connection.setRequestMethod(method); 86 connection.setRequestProperty(HDR_ACCEPT, 87 Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); 88 connection.setRequestProperty(HDR_CONTENT_TYPE, 89 Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); 90 additionalHeaders 91 .forEach((k, v) -> connection.setRequestProperty(k, v)); 92 return connection; 93 } 94 getLfsUrl(Repository db, String purpose, Map<String, String> additionalHeaders)95 private static String getLfsUrl(Repository db, String purpose, 96 Map<String, String> additionalHeaders) 97 throws LfsConfigInvalidException { 98 StoredConfig config = db.getConfig(); 99 String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, 100 null, 101 ConfigConstants.CONFIG_KEY_URL); 102 Exception ex = null; 103 if (lfsUrl == null) { 104 String remoteUrl = null; 105 for (String remote : db.getRemoteNames()) { 106 lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, 107 remote, 108 ConfigConstants.CONFIG_KEY_URL); 109 // This could be done better (more precise logic), but according 110 // to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs 111 // generally only supports 'origin' in an integrated workflow. 112 if (lfsUrl == null && (remote.equals( 113 org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) { 114 remoteUrl = config.getString( 115 ConfigConstants.CONFIG_KEY_REMOTE, remote, 116 ConfigConstants.CONFIG_KEY_URL); 117 break; 118 } 119 } 120 if (lfsUrl == null && remoteUrl != null) { 121 try { 122 lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders, 123 remoteUrl); 124 } catch (URISyntaxException | IOException 125 | CommandFailedException e) { 126 ex = e; 127 } 128 } else { 129 lfsUrl = lfsUrl + Protocol.INFO_LFS_ENDPOINT; 130 } 131 } 132 if (lfsUrl == null) { 133 if (ex != null) { 134 throw new LfsConfigInvalidException( 135 LfsText.get().lfsNoDownloadUrl, ex); 136 } 137 throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl); 138 } 139 return lfsUrl; 140 } 141 discoverLfsUrl(Repository db, String purpose, Map<String, String> additionalHeaders, String remoteUrl)142 private static String discoverLfsUrl(Repository db, String purpose, 143 Map<String, String> additionalHeaders, String remoteUrl) 144 throws URISyntaxException, IOException, CommandFailedException { 145 URIish u = new URIish(remoteUrl); 146 if (u.getScheme() == null || SCHEME_SSH.equals(u.getScheme())) { 147 Protocol.ExpiringAction action = getSshAuthentication(db, purpose, 148 remoteUrl, u); 149 additionalHeaders.putAll(action.header); 150 return action.href; 151 } 152 return remoteUrl + Protocol.INFO_LFS_ENDPOINT; 153 } 154 getSshAuthentication( Repository db, String purpose, String remoteUrl, URIish u)155 private static Protocol.ExpiringAction getSshAuthentication( 156 Repository db, String purpose, String remoteUrl, URIish u) 157 throws IOException, CommandFailedException { 158 AuthCache cached = sshAuthCache.get(remoteUrl); 159 Protocol.ExpiringAction action = null; 160 if (cached != null && cached.validUntil > System.currentTimeMillis()) { 161 action = cached.cachedAction; 162 } 163 164 if (action == null) { 165 // discover and authenticate; git-lfs does "ssh 166 // -p <port> -- <host> git-lfs-authenticate 167 // <project> <upload/download>" 168 String json = SshSupport.runSshCommand(u.setPath(""), //$NON-NLS-1$ 169 null, db.getFS(), 170 "git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$ 171 + purpose, 172 SSH_AUTH_TIMEOUT_SECONDS); 173 174 action = Protocol.gson().fromJson(json, 175 Protocol.ExpiringAction.class); 176 177 // cache the result as long as possible. 178 AuthCache c = new AuthCache(action); 179 sshAuthCache.put(remoteUrl, c); 180 } 181 return action; 182 } 183 184 /** 185 * Create a connection for the specified 186 * {@link org.eclipse.jgit.lfs.Protocol.Action}. 187 * 188 * @param repo 189 * the repo to fetch required configuration from 190 * @param action 191 * the action for which to create a connection 192 * @param method 193 * the target method (GET or PUT) 194 * @return a connection. output mode is not set. 195 * @throws IOException 196 * in case of any error. 197 */ 198 @NonNull getLfsContentConnection( Repository repo, Protocol.Action action, String method)199 public static HttpConnection getLfsContentConnection( 200 Repository repo, Protocol.Action action, String method) 201 throws IOException { 202 URL contentUrl = new URL(action.href); 203 HttpConnection contentServerConn = HttpTransport.getConnectionFactory() 204 .create(contentUrl, HttpSupport 205 .proxyFor(ProxySelector.getDefault(), contentUrl)); 206 contentServerConn.setRequestMethod(method); 207 if (action.header != null) { 208 action.header.forEach( 209 (k, v) -> contentServerConn.setRequestProperty(k, v)); 210 } 211 if (contentUrl.getProtocol().equals(SCHEME_HTTPS) 212 && !repo.getConfig().getBoolean(HttpConfig.HTTP, 213 HttpConfig.SSL_VERIFY_KEY, true)) { 214 HttpSupport.disableSslVerify(contentServerConn); 215 } 216 217 contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING, 218 ENCODING_GZIP); 219 220 return contentServerConn; 221 } 222 extractProjectName(URIish u)223 private static String extractProjectName(URIish u) { 224 String path = u.getPath(); 225 226 // begins with a slash if the url contains a port (gerrit vs. github). 227 if (path.startsWith("/")) { //$NON-NLS-1$ 228 path = path.substring(1); 229 } 230 231 if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) { 232 return path.substring(0, path.length() - 4); 233 } 234 return path; 235 } 236 237 /** 238 * @param operation 239 * the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD 240 * @param resources 241 * the LFS resources affected 242 * @return a request that can be serialized to JSON 243 */ toRequest(String operation, LfsPointer... resources)244 public static Protocol.Request toRequest(String operation, 245 LfsPointer... resources) { 246 Protocol.Request req = new Protocol.Request(); 247 req.operation = operation; 248 if (resources != null) { 249 req.objects = new LinkedList<>(); 250 for (LfsPointer res : resources) { 251 Protocol.ObjectSpec o = new Protocol.ObjectSpec(); 252 o.oid = res.getOid().getName(); 253 o.size = res.getSize(); 254 req.objects.add(o); 255 } 256 } 257 return req; 258 } 259 260 private static final class AuthCache { 261 private static final long AUTH_CACHE_EAGER_TIMEOUT = 500; 262 263 private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter 264 .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$ 265 266 /** 267 * Creates a cache entry for an authentication response. 268 * <p> 269 * The timeout of the cache token is extracted from the given action. If 270 * no timeout can be determined, the token will be used only once. 271 * 272 * @param action 273 */ AuthCache(Protocol.ExpiringAction action)274 public AuthCache(Protocol.ExpiringAction action) { 275 this.cachedAction = action; 276 try { 277 if (action.expiresIn != null && !action.expiresIn.isEmpty()) { 278 this.validUntil = (System.currentTimeMillis() 279 + Long.parseLong(action.expiresIn)) 280 - AUTH_CACHE_EAGER_TIMEOUT; 281 } else if (action.expiresAt != null 282 && !action.expiresAt.isEmpty()) { 283 this.validUntil = LocalDateTime 284 .parse(action.expiresAt, ISO_FORMAT) 285 .atZone(ZoneOffset.UTC).toInstant().toEpochMilli() 286 - AUTH_CACHE_EAGER_TIMEOUT; 287 } else { 288 this.validUntil = System.currentTimeMillis(); 289 } 290 } catch (Exception e) { 291 this.validUntil = System.currentTimeMillis(); 292 } 293 } 294 295 long validUntil; 296 297 Protocol.ExpiringAction cachedAction; 298 } 299 300 } 301