1 /* 2 * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others 3 * 4 * This program and the accompanying materials are made available under the 5 * terms of the Eclipse Distribution License v. 1.0 which is available at 6 * https://www.eclipse.org/org/documents/edl-v10.php. 7 * 8 * SPDX-License-Identifier: BSD-3-Clause 9 */ 10 package org.eclipse.jgit.lfs.server.fs; 11 12 import static java.nio.charset.StandardCharsets.UTF_8; 13 import static org.junit.Assert.assertEquals; 14 15 import java.io.BufferedInputStream; 16 import java.io.FileNotFoundException; 17 import java.io.IOException; 18 import java.io.InputStream; 19 import java.nio.ByteBuffer; 20 import java.nio.channels.Channels; 21 import java.nio.channels.FileChannel; 22 import java.nio.channels.ReadableByteChannel; 23 import java.nio.file.Files; 24 import java.nio.file.Path; 25 import java.nio.file.Paths; 26 import java.nio.file.StandardOpenOption; 27 import java.security.DigestInputStream; 28 import java.security.SecureRandom; 29 30 import org.apache.http.HttpEntity; 31 import org.apache.http.HttpResponse; 32 import org.apache.http.StatusLine; 33 import org.apache.http.client.ClientProtocolException; 34 import org.apache.http.client.methods.CloseableHttpResponse; 35 import org.apache.http.client.methods.HttpGet; 36 import org.apache.http.client.methods.HttpPut; 37 import org.apache.http.entity.ContentType; 38 import org.apache.http.entity.InputStreamEntity; 39 import org.apache.http.entity.StringEntity; 40 import org.apache.http.impl.client.CloseableHttpClient; 41 import org.apache.http.impl.client.HttpClientBuilder; 42 import org.eclipse.jetty.servlet.ServletContextHandler; 43 import org.eclipse.jetty.servlet.ServletHolder; 44 import org.eclipse.jgit.junit.MockSystemReader; 45 import org.eclipse.jgit.junit.http.AppServer; 46 import org.eclipse.jgit.lfs.errors.LfsException; 47 import org.eclipse.jgit.lfs.lib.AnyLongObjectId; 48 import org.eclipse.jgit.lfs.lib.Constants; 49 import org.eclipse.jgit.lfs.lib.LongObjectId; 50 import org.eclipse.jgit.lfs.server.LargeFileRepository; 51 import org.eclipse.jgit.lfs.server.LfsProtocolServlet; 52 import org.eclipse.jgit.lfs.test.LongObjectIdTestUtils; 53 import org.eclipse.jgit.util.FS; 54 import org.eclipse.jgit.util.FileUtils; 55 import org.eclipse.jgit.util.IO; 56 import org.eclipse.jgit.util.SystemReader; 57 import org.junit.After; 58 import org.junit.Before; 59 60 public abstract class LfsServerTest { 61 62 private static final long timeout = /* 10 sec */ 10 * 1000; 63 64 protected static final int MiB = 1024 * 1024; 65 66 /** In-memory application server; subclass must start. */ 67 protected AppServer server; 68 69 private Path tmp; 70 71 private Path dir; 72 73 protected FileLfsRepository repository; 74 75 protected FileLfsServlet servlet; 76 LfsServerTest()77 public LfsServerTest() { 78 super(); 79 } 80 getTempDirectory()81 public Path getTempDirectory() { 82 return tmp; 83 } 84 getDir()85 public Path getDir() { 86 return dir; 87 } 88 89 @Before setup()90 public void setup() throws Exception { 91 SystemReader.setInstance(new MockSystemReader()); 92 tmp = Files.createTempDirectory("jgit_test_"); 93 94 // measure timer resolution before the test to avoid time critical tests 95 // are affected by time needed for measurement 96 FS.getFileStoreAttributes(tmp.getParent()); 97 98 server = new AppServer(); 99 ServletContextHandler app = server.addContext("/lfs"); 100 dir = Paths.get(tmp.toString(), "lfs"); 101 this.repository = new FileLfsRepository(null, dir); 102 servlet = new FileLfsServlet(repository, timeout); 103 app.addServlet(new ServletHolder(servlet), "/objects/*"); 104 105 LfsProtocolServlet protocol = new LfsProtocolServlet() { 106 private static final long serialVersionUID = 1L; 107 108 @Override 109 protected LargeFileRepository getLargeFileRepository( 110 LfsRequest request, String path, String auth) 111 throws LfsException { 112 return repository; 113 } 114 }; 115 app.addServlet(new ServletHolder(protocol), "/objects/batch"); 116 117 server.setUp(); 118 this.repository.setUrl(server.getURI() + "/lfs/objects/"); 119 } 120 121 @After tearDown()122 public void tearDown() throws Exception { 123 server.tearDown(); 124 FileUtils.delete(tmp.toFile(), FileUtils.RECURSIVE | FileUtils.RETRY); 125 } 126 putContent(String s)127 protected AnyLongObjectId putContent(String s) 128 throws IOException, ClientProtocolException { 129 AnyLongObjectId id = LongObjectIdTestUtils.hash(s); 130 return putContent(id, s); 131 } 132 putContent(AnyLongObjectId id, String s)133 protected AnyLongObjectId putContent(AnyLongObjectId id, String s) 134 throws ClientProtocolException, IOException { 135 try (CloseableHttpClient client = HttpClientBuilder.create().build()) { 136 HttpEntity entity = new StringEntity(s, 137 ContentType.APPLICATION_OCTET_STREAM); 138 String hexId = id.name(); 139 HttpPut request = new HttpPut( 140 server.getURI() + "/lfs/objects/" + hexId); 141 request.setEntity(entity); 142 try (CloseableHttpResponse response = client.execute(request)) { 143 StatusLine statusLine = response.getStatusLine(); 144 int status = statusLine.getStatusCode(); 145 if (status >= 400) { 146 throw new RuntimeException("Status: " + status + ". " 147 + statusLine.getReasonPhrase()); 148 } 149 } 150 return id; 151 } 152 } 153 putContent(Path f)154 protected LongObjectId putContent(Path f) 155 throws FileNotFoundException, IOException { 156 try (CloseableHttpClient client = HttpClientBuilder.create().build()) { 157 LongObjectId id1, id2; 158 String hexId1, hexId2; 159 try (DigestInputStream in = new DigestInputStream( 160 new BufferedInputStream(Files.newInputStream(f)), 161 Constants.newMessageDigest())) { 162 InputStreamEntity entity = new InputStreamEntity(in, 163 Files.size(f), ContentType.APPLICATION_OCTET_STREAM); 164 id1 = LongObjectIdTestUtils.hash(f); 165 hexId1 = id1.name(); 166 HttpPut request = new HttpPut( 167 server.getURI() + "/lfs/objects/" + hexId1); 168 request.setEntity(entity); 169 HttpResponse response = client.execute(request); 170 checkResponseStatus(response); 171 id2 = LongObjectId.fromRaw(in.getMessageDigest().digest()); 172 hexId2 = id2.name(); 173 assertEquals(hexId1, hexId2); 174 } 175 return id1; 176 } 177 } 178 checkResponseStatus(HttpResponse response)179 private void checkResponseStatus(HttpResponse response) { 180 StatusLine statusLine = response.getStatusLine(); 181 int status = statusLine.getStatusCode(); 182 if (statusLine.getStatusCode() >= 400) { 183 String error; 184 try { 185 ByteBuffer buf = IO.readWholeStream(new BufferedInputStream( 186 response.getEntity().getContent()), 1024); 187 if (buf.hasArray()) { 188 error = new String(buf.array(), 189 buf.arrayOffset() + buf.position(), buf.remaining(), 190 UTF_8); 191 } else { 192 final byte[] b = new byte[buf.remaining()]; 193 buf.duplicate().get(b); 194 error = new String(b, UTF_8); 195 } 196 } catch (IOException e) { 197 error = statusLine.getReasonPhrase(); 198 } 199 throw new RuntimeException("Status: " + status + " " + error); 200 } 201 assertEquals(200, status); 202 } 203 getContent(AnyLongObjectId id, Path f)204 protected long getContent(AnyLongObjectId id, Path f) throws IOException { 205 String hexId = id.name(); 206 return getContent(hexId, f); 207 } 208 getContent(String hexId, Path f)209 protected long getContent(String hexId, Path f) throws IOException { 210 try (CloseableHttpClient client = HttpClientBuilder.create().build()) { 211 HttpGet request = new HttpGet( 212 server.getURI() + "/lfs/objects/" + hexId); 213 HttpResponse response = client.execute(request); 214 checkResponseStatus(response); 215 HttpEntity entity = response.getEntity(); 216 long pos = 0; 217 try (InputStream in = entity.getContent(); 218 ReadableByteChannel inChannel = Channels.newChannel(in); 219 FileChannel outChannel = FileChannel.open(f, 220 StandardOpenOption.CREATE_NEW, 221 StandardOpenOption.WRITE)) { 222 long transferred; 223 do { 224 transferred = outChannel.transferFrom(inChannel, pos, MiB); 225 pos += transferred; 226 } while (transferred > 0); 227 } 228 return pos; 229 } 230 } 231 232 /** 233 * Creates a file with random content, repeatedly writing a random string of 234 * 4k length to the file until the file has at least the specified length. 235 * 236 * @param f 237 * file to fill 238 * @param size 239 * size of the file to generate 240 * @return length of the generated file in bytes 241 * @throws IOException 242 */ createPseudoRandomContentFile(Path f, long size)243 protected long createPseudoRandomContentFile(Path f, long size) 244 throws IOException { 245 SecureRandom rnd = new SecureRandom(); 246 byte[] buf = new byte[4096]; 247 rnd.nextBytes(buf); 248 ByteBuffer bytebuf = ByteBuffer.wrap(buf); 249 try (FileChannel outChannel = FileChannel.open(f, 250 StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { 251 long len = 0; 252 do { 253 len += outChannel.write(bytebuf); 254 if (bytebuf.position() == 4096) { 255 bytebuf.rewind(); 256 } 257 } while (len < size); 258 } 259 return Files.size(f); 260 } 261 } 262