1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * See LICENSE.txt included in this distribution for the specific 9 * language governing permissions and limitations under the License. 10 * 11 * When distributing Covered Code, include this CDDL HEADER in each 12 * file and include the License file at LICENSE.txt. 13 * If applicable, add the following below this CDDL HEADER, with the 14 * fields enclosed by brackets "[]" replaced with your own identifying 15 * information: Portions Copyright [yyyy] [name of copyright owner] 16 * 17 * CDDL HEADER END 18 */ 19 20 /* 21 * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved. 22 * Portions Copyright (c) 2018, 2020, Chris Fraire <cfraire@me.com>. 23 * Portions Copyright (c) 2020, Ric Harris <harrisric@users.noreply.github.com>. 24 */ 25 package org.opengrok.indexer.history; 26 27 import static org.junit.jupiter.api.Assertions.assertEquals; 28 import static org.junit.jupiter.api.Assertions.assertFalse; 29 import static org.junit.jupiter.api.Assertions.assertNotNull; 30 import static org.junit.jupiter.api.Assertions.assertNull; 31 import static org.junit.jupiter.api.Assertions.assertThrows; 32 import static org.junit.jupiter.api.Assertions.assertTrue; 33 import static org.opengrok.indexer.condition.RepositoryInstalled.Type.MERCURIAL; 34 import static org.opengrok.indexer.condition.RepositoryInstalled.Type.SCCS; 35 import static org.opengrok.indexer.condition.RepositoryInstalled.Type.SUBVERSION; 36 import static org.opengrok.indexer.history.MercurialRepositoryTest.runHgCommand; 37 38 import java.io.File; 39 import java.io.FileOutputStream; 40 import java.io.IOException; 41 import java.nio.charset.StandardCharsets; 42 import java.nio.file.Files; 43 import java.nio.file.Path; 44 import java.nio.file.Paths; 45 import java.nio.file.attribute.FileTime; 46 import java.util.Date; 47 import java.util.Iterator; 48 import java.util.LinkedList; 49 import java.util.List; 50 import java.util.Map; 51 52 import org.apache.commons.lang3.time.DateUtils; 53 import org.eclipse.jgit.api.Git; 54 import org.junit.jupiter.api.AfterEach; 55 import org.junit.jupiter.api.BeforeEach; 56 import org.junit.jupiter.api.Test; 57 import org.junit.jupiter.api.condition.EnabledOnOs; 58 import org.junit.jupiter.api.condition.OS; 59 import org.junit.jupiter.params.ParameterizedTest; 60 import org.junit.jupiter.params.provider.ValueSource; 61 import org.mockito.Mockito; 62 import org.opengrok.indexer.condition.EnabledForRepository; 63 import org.opengrok.indexer.configuration.Filter; 64 import org.opengrok.indexer.configuration.IgnoredNames; 65 import org.opengrok.indexer.configuration.RuntimeEnvironment; 66 import org.opengrok.indexer.util.IOUtils; 67 import org.opengrok.indexer.util.TestRepository; 68 69 /** 70 * Test file based history cache with special focus on incremental reindex. 71 * 72 * @author Vladimir Kotal 73 */ 74 class FileHistoryCacheTest { 75 76 private static final String SVN_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; 77 78 private final RuntimeEnvironment env = RuntimeEnvironment.getInstance(); 79 private TestRepository repositories; 80 private FileHistoryCache cache; 81 82 private boolean savedFetchHistoryWhenNotInCache; 83 private boolean savedIsHandleHistoryOfRenamedFiles; 84 private boolean savedIsTagsEnabled; 85 86 /** 87 * Set up the test environment with repositories and a cache instance. 88 */ 89 @BeforeEach setUp()90 public void setUp() throws Exception { 91 repositories = new TestRepository(); 92 repositories.create(getClass().getResource("/repositories")); 93 94 // Needed for HistoryGuru to operate normally. 95 env.setRepositories(repositories.getSourceRoot()); 96 97 cache = new FileHistoryCache(); 98 cache.initialize(); 99 100 savedFetchHistoryWhenNotInCache = env.isFetchHistoryWhenNotInCache(); 101 savedIsHandleHistoryOfRenamedFiles = env.isHandleHistoryOfRenamedFiles(); 102 savedIsTagsEnabled = env.isTagsEnabled(); 103 } 104 105 /** 106 * Clean up after the test. Remove the test repositories. 107 */ 108 @AfterEach tearDown()109 public void tearDown() { 110 repositories.destroy(); 111 repositories = null; 112 113 cache = null; 114 115 env.setFetchHistoryWhenNotInCache(savedFetchHistoryWhenNotInCache); 116 env.setIgnoredNames(new IgnoredNames()); 117 env.setIncludedNames(new Filter()); 118 env.setHandleHistoryOfRenamedFiles(savedIsHandleHistoryOfRenamedFiles); 119 env.setTagsEnabled(savedIsTagsEnabled); 120 } 121 122 /** 123 * Assert that two lists of HistoryEntry objects are equal. 124 * 125 * @param expected the expected list of entries 126 * @param actual the actual list of entries 127 * @param isdir was the history generated for a directory 128 * @throws AssertionError if the two lists don't match 129 */ assertSameEntries(List<HistoryEntry> expected, List<HistoryEntry> actual, boolean isdir)130 private void assertSameEntries(List<HistoryEntry> expected, List<HistoryEntry> actual, boolean isdir) { 131 assertEquals(expected.size(), actual.size(), "Unexpected size"); 132 Iterator<HistoryEntry> actualIt = actual.iterator(); 133 for (HistoryEntry expectedEntry : expected) { 134 assertSameEntry(expectedEntry, actualIt.next(), isdir); 135 } 136 assertFalse(actualIt.hasNext(), "More entries than expected"); 137 } 138 139 /** 140 * Assert that two HistoryEntry objects are equal. 141 * 142 * @param expected the expected instance 143 * @param actual the actual instance 144 * @param isdir was the history generated for directory 145 * @throws AssertionError if the two instances don't match 146 */ assertSameEntry(HistoryEntry expected, HistoryEntry actual, boolean isdir)147 private void assertSameEntry(HistoryEntry expected, HistoryEntry actual, boolean isdir) { 148 assertEquals(expected.getAuthor(), actual.getAuthor()); 149 assertEquals(expected.getRevision(), actual.getRevision()); 150 assertEquals(expected.getDate(), actual.getDate()); 151 assertEquals(expected.getMessage(), actual.getMessage()); 152 if (isdir) { 153 assertEquals(expected.getFiles().size(), actual.getFiles().size()); 154 } else { 155 assertEquals(0, actual.getFiles().size()); 156 } 157 } 158 159 /** 160 * {@link FileHistoryCache#get(File, Repository, boolean)} should not disturb history cache 161 * if run between repository update and reindex. 162 */ 163 @EnabledOnOs({OS.LINUX, OS.MAC, OS.SOLARIS, OS.AIX, OS.OTHER}) 164 @EnabledForRepository(MERCURIAL) 165 @Test testStoreTouchGet()166 void testStoreTouchGet() throws Exception { 167 File reposRoot = new File(repositories.getSourceRoot(), "mercurial"); 168 Repository repo = RepositoryFactory.getRepository(reposRoot); 169 History historyToStore = repo.getHistory(reposRoot); 170 171 cache.store(historyToStore, repo); 172 173 // This makes sure that the file which contains the latest revision has indeed been created. 174 assertEquals("9:8b340409b3a8", cache.getLatestCachedRevision(repo)); 175 176 File file = new File(reposRoot, "main.c"); 177 assertTrue(file.exists()); 178 FileTime fileTimeBeforeImport = Files.getLastModifiedTime(file.toPath()); 179 History historyBeforeImport = cache.get(file, repo, false); 180 181 MercurialRepositoryTest.runHgCommand(reposRoot, "import", 182 Paths.get(getClass().getResource("/history/hg-export.txt").toURI()).toString()); 183 FileTime fileTimeAfterImport = Files.getLastModifiedTime(file.toPath()); 184 assertTrue(fileTimeBeforeImport.compareTo(fileTimeAfterImport) < 0); 185 186 // Simulates reindex, or at least its first part when history cache is updated. 187 repo.createCache(cache, cache.getLatestCachedRevision(repo)); 188 189 // This makes sure that the file which contains the latest revision has indeed been created. 190 assertEquals("11:bbb3ce75e1b8", cache.getLatestCachedRevision(repo)); 191 192 /* 193 * The history should not be disturbed. 194 * Make sure that get() retrieved the history from cache. Mocking/spying static methods 195 * (FileHistoryCache#readCache() in this case) is tricky so use the cache hits metric. 196 */ 197 double cacheHitsBeforeGet = cache.getFileHistoryCacheHits(); 198 History historyAfterReindex = cache.get(file, repo, false); 199 double cacheHitsAfterGet = cache.getFileHistoryCacheHits(); 200 assertNotNull(historyAfterReindex); 201 assertEquals(1, cacheHitsAfterGet - cacheHitsBeforeGet); 202 } 203 204 /** 205 * Basic tests for the {@code store()} method on cache with disabled 206 * handling of renamed files. 207 */ 208 @EnabledForRepository(MERCURIAL) 209 @Test 210 void testStoreAndGetNotRenamed() throws Exception { 211 File reposRoot = new File(repositories.getSourceRoot(), "mercurial"); 212 Repository repo = RepositoryFactory.getRepository(reposRoot); 213 History historyToStore = repo.getHistory(reposRoot); 214 215 cache.store(historyToStore, repo); 216 217 // This makes sure that the file which contains the latest revision 218 // has indeed been created. 219 assertEquals("9:8b340409b3a8", cache.getLatestCachedRevision(repo)); 220 221 // test reindex 222 History historyNull = new History(); 223 cache.store(historyNull, repo); 224 225 assertEquals("9:8b340409b3a8", cache.getLatestCachedRevision(repo)); 226 } 227 228 /** 229 * Test tagging by creating history cache for repository with one tag and 230 * then importing couple of changesets which add both file changes and tags. 231 * The last history entry before the import is important as it needs to be 232 * retagged when old history is merged with the new one. 233 */ 234 @EnabledForRepository(MERCURIAL) 235 @Test 236 void testStoreAndGetIncrementalTags() throws Exception { 237 // Enable tagging of history entries. 238 env.setTagsEnabled(true); 239 240 File reposRoot = new File(repositories.getSourceRoot(), "mercurial"); 241 Repository repo = RepositoryFactory.getRepository(reposRoot); 242 History historyToStore = repo.getHistory(reposRoot); 243 244 // Store the history. 245 cache.store(historyToStore, repo); 246 247 // Avoid uncommitted changes. 248 MercurialRepositoryTest.runHgCommand(reposRoot, "revert", "--all"); 249 250 // Add bunch of changesets with file based changes and tags. 251 MercurialRepositoryTest.runHgCommand(reposRoot, "import", 252 Paths.get(getClass().getResource("/history/hg-export-tag.txt").toURI()).toString()); 253 254 // Perform incremental reindex. 255 repo.createCache(cache, cache.getLatestCachedRevision(repo)); 256 257 // Verify tags in fileHistory for main.c which is the most interesting 258 // file from the repository from the perspective of tags. 259 File main = new File(reposRoot, "main.c"); 260 assertTrue(main.exists()); 261 History retrievedHistoryMainC = cache.get(main, repo, true); 262 List<HistoryEntry> entries = retrievedHistoryMainC.getHistoryEntries(); 263 assertEquals(3, entries.size(), "Unexpected number of entries for main.c"); 264 HistoryEntry e0 = entries.get(0); 265 assertEquals("13:3d386f6bd848", e0.getRevision(), "Unexpected revision for entry 0"); 266 assertEquals("tag3", retrievedHistoryMainC.getTags().get(e0.getRevision()), 267 "Invalid tag list for revision 13"); 268 HistoryEntry e1 = entries.get(1); 269 assertEquals("2:585a1b3f2efb", e1.getRevision(), "Unexpected revision for entry 1"); 270 assertEquals("tag2, tag1, start_of_novel", retrievedHistoryMainC.getTags().get(e1.getRevision()), 271 "Invalid tag list for revision 2"); 272 HistoryEntry e2 = entries.get(2); 273 assertEquals("1:f24a5fd7a85d", e2.getRevision(), "Unexpected revision for entry 2"); 274 assertNull(retrievedHistoryMainC.getTags().get(e2.getRevision()), "Invalid tag list for revision 1"); 275 276 // Reindex from scratch. 277 String histCachePath = FileHistoryCache.getRepositoryHistDataDirname(repo); 278 assertNotNull(histCachePath); 279 File dir = new File(histCachePath); 280 assertTrue(dir.isDirectory()); 281 cache.clear(repo); 282 assertFalse(dir.exists()); 283 History freshHistory = repo.getHistory(reposRoot); 284 cache.store(freshHistory, repo); 285 assertTrue(dir.exists()); 286 287 History retrievedUpdatedHistoryMainC = cache.get(main, repo, true); 288 assertSameEntries(retrievedHistoryMainC.getHistoryEntries(), 289 retrievedUpdatedHistoryMainC.getHistoryEntries(), false); 290 assertEquals(Map.of("13:3d386f6bd848", "tag3", "2:585a1b3f2efb", "tag2, tag1, start_of_novel"), 291 retrievedUpdatedHistoryMainC.getTags()); 292 } 293 294 /** 295 * Basic tests for the {@code store()} and {@code get()} methods. 296 */ 297 @Test 298 @EnabledOnOs({OS.LINUX, OS.MAC, OS.SOLARIS, OS.AIX, OS.OTHER}) 299 @EnabledForRepository(MERCURIAL) 300 void testStoreAndGet() throws Exception { 301 File reposRoot = new File(repositories.getSourceRoot(), "mercurial"); 302 303 // The test expects support for renamed files. 304 env.setHandleHistoryOfRenamedFiles(true); 305 306 Repository repo = RepositoryFactory.getRepository(reposRoot); 307 History historyToStore = repo.getHistory(reposRoot); 308 309 cache.store(historyToStore, repo); 310 311 // test reindex 312 History historyNull = new History(); 313 cache.store(historyNull, repo); 314 315 // test get history for single file 316 File makefile = new File(reposRoot, "Makefile"); 317 assertTrue(makefile.exists()); 318 319 History retrievedHistory = cache.get(makefile, repo, true); 320 321 List<HistoryEntry> entries = retrievedHistory.getHistoryEntries(); 322 323 assertEquals(2, entries.size(), "Unexpected number of entries"); 324 325 final String TROND = "Trond Norbye <trond.norbye@sun.com>"; 326 327 Iterator<HistoryEntry> entryIt = entries.iterator(); 328 329 HistoryEntry e1 = entryIt.next(); 330 assertEquals(TROND, e1.getAuthor()); 331 assertEquals("2:585a1b3f2efb", e1.getRevision()); 332 assertEquals(0, e1.getFiles().size()); 333 334 HistoryEntry e2 = entryIt.next(); 335 assertEquals(TROND, e2.getAuthor()); 336 assertEquals("1:f24a5fd7a85d", e2.getRevision()); 337 assertEquals(0, e2.getFiles().size()); 338 339 assertFalse(entryIt.hasNext()); 340 341 // test get history for renamed file 342 File novel = new File(reposRoot, "novel.txt"); 343 assertTrue(novel.exists()); 344 345 retrievedHistory = cache.get(novel, repo, true); 346 347 entries = retrievedHistory.getHistoryEntries(); 348 349 assertEquals(6, entries.size(), "Unexpected number of entries"); 350 351 // test incremental update 352 MercurialRepositoryTest.runHgCommand(reposRoot, "import", 353 Paths.get(getClass().getResource("/history/hg-export.txt").toURI()).toString()); 354 355 repo.createCache(cache, cache.getLatestCachedRevision(repo)); 356 357 File mainC = new File(reposRoot, "main.c"); 358 History updatedHistory = cache.get(mainC, repo, false); 359 assertNotNull(updatedHistory); 360 361 HistoryEntry newEntry1 = new HistoryEntry( 362 "10:1e392ef0b0ed", 363 new Date(1245446973L / 60 * 60 * 1000), // whole minutes only 364 "xyz", 365 "Return failure when executed with no arguments", 366 true); 367 HistoryEntry newEntry2 = new HistoryEntry( 368 "11:bbb3ce75e1b8", 369 new Date(1245447973L / 60 * 60 * 1000), // whole minutes only 370 "xyz", 371 "Do something else", 372 true); 373 374 LinkedList<HistoryEntry> updatedEntries = new LinkedList<>( 375 updatedHistory.getHistoryEntries()); 376 assertSameEntry(newEntry2, updatedEntries.removeFirst(), false); 377 assertSameEntry(newEntry1, updatedEntries.removeFirst(), false); 378 379 // test clearing of cache 380 String dirPath = FileHistoryCache.getRepositoryHistDataDirname(repo); 381 assertNotNull(dirPath); 382 File dir = new File(dirPath); 383 assertTrue(dir.isDirectory()); 384 cache.clear(repo); 385 assertFalse(dir.exists()); 386 387 cache.store(historyToStore, repo); 388 // check that the data directory is non-empty 389 assertTrue(dir.list().length > 0); 390 } 391 392 /** 393 * Check how incremental reindex behaves when indexing changesets that 394 * rename+change file. 395 * 396 * The scenario goes as follows: 397 * - create Mercurial repository 398 * - perform full reindex 399 * - add changesets which renamed and modify a file 400 * - perform incremental reindex 401 * - change+rename the file again 402 * - incremental reindex 403 */ 404 @EnabledOnOs({OS.LINUX, OS.MAC, OS.SOLARIS, OS.AIX, OS.OTHER}) 405 @EnabledForRepository(MERCURIAL) 406 @Test testRenameFileThenDoIncrementalReindex()407 void testRenameFileThenDoIncrementalReindex() throws Exception { 408 File reposRoot = new File(repositories.getSourceRoot(), "mercurial"); 409 History updatedHistory; 410 411 // The test expects support for renamed files. 412 env.setHandleHistoryOfRenamedFiles(true); 413 414 // Use tags for better coverage. 415 env.setTagsEnabled(true); 416 417 // Generate history index. 418 // It is necessary to call getRepository() only after tags were enabled 419 // to produce list of tags. 420 Repository repo = RepositoryFactory.getRepository(reposRoot); 421 History historyToStore = repo.getHistory(reposRoot); 422 cache.store(historyToStore, repo); 423 424 // Import changesets which rename one of the files in the repository. 425 MercurialRepositoryTest.runHgCommand(reposRoot, "import", 426 Paths.get(getClass().getResource("/history/hg-export-renamed.txt").toURI()).toString()); 427 428 // Perform incremental reindex. 429 repo.createCache(cache, cache.getLatestCachedRevision(repo)); 430 431 // Check changesets for the renames and changes of single file. 432 File main2File = new File(reposRoot.toString() + File.separatorChar + "main2.c"); 433 updatedHistory = cache.get(main2File, repo, false); 434 435 // Changesets e0-e3 were brought in by the import done above. 436 HistoryEntry e0 = new HistoryEntry( 437 "13:e55a793086da", 438 new Date(1245447973L / 60 * 60 * 1000), // whole minutes only 439 "xyz", 440 "Do something else", 441 true); 442 HistoryEntry e1 = new HistoryEntry( 443 "12:97b5392fec0d", 444 new Date(1393515253L / 60 * 60 * 1000), // whole minutes only 445 "Vladimir Kotal <Vladimir.Kotal@oracle.com>", 446 "rename2", 447 true); 448 HistoryEntry e2 = new HistoryEntry( 449 "11:5c203a0bc12b", 450 new Date(1393515291L / 60 * 60 * 1000), // whole minutes only 451 "Vladimir Kotal <Vladimir.Kotal@oracle.com>", 452 "rename1", 453 true); 454 HistoryEntry e3 = new HistoryEntry( 455 "10:1e392ef0b0ed", 456 new Date(1245446973L / 60 * 60 * 1000), // whole minutes only 457 "xyz", 458 "Return failure when executed with no arguments", 459 true); 460 HistoryEntry e4 = new HistoryEntry( 461 "2:585a1b3f2efb", 462 new Date(1218571989L / 60 * 60 * 1000), // whole minutes only 463 "Trond Norbye <trond.norbye@sun.com>", 464 "Add lint make target and fix lint warnings", 465 true); 466 HistoryEntry e5 = new HistoryEntry( 467 "1:f24a5fd7a85d", 468 new Date(1218571413L / 60 * 60 * 1000), // whole minutes only 469 "Trond Norbye <trond.norbye@sun.com>", 470 "Created a small dummy program", 471 true); 472 473 History histConstruct = new History(); 474 LinkedList<HistoryEntry> entriesConstruct = new LinkedList<>(); 475 entriesConstruct.add(e0); 476 entriesConstruct.add(e1); 477 entriesConstruct.add(e2); 478 entriesConstruct.add(e3); 479 entriesConstruct.add(e4); 480 entriesConstruct.add(e5); 481 histConstruct.setHistoryEntries(entriesConstruct); 482 assertEquals(6, updatedHistory.getHistoryEntries().size()); 483 assertSameEntries(histConstruct.getHistoryEntries(), 484 updatedHistory.getHistoryEntries(), false); 485 486 // Add some changes and rename the file again. 487 MercurialRepositoryTest.runHgCommand(reposRoot, "import", 488 Paths.get(getClass().getResource("/history/hg-export-renamed-again.txt").toURI()).toString()); 489 490 // Perform incremental reindex. 491 repo.createCache(cache, cache.getLatestCachedRevision(repo)); 492 493 HistoryEntry e6 = new HistoryEntry( 494 "14:55c41cd4b348", 495 new Date(1489505558L / 60 * 60 * 1000), // whole minutes only 496 "Vladimir Kotal <Vladimir.Kotal@oracle.com>", "rename + cstyle", 497 true); 498 entriesConstruct = new LinkedList<>(); 499 entriesConstruct.add(e6); 500 entriesConstruct.add(e0); 501 entriesConstruct.add(e1); 502 entriesConstruct.add(e2); 503 entriesConstruct.add(e3); 504 entriesConstruct.add(e4); 505 entriesConstruct.add(e5); 506 histConstruct.setHistoryEntries(entriesConstruct); 507 508 // Check changesets for the renames and changes of single file. 509 File main3File = new File(reposRoot.toString() + File.separatorChar + "main3.c"); 510 updatedHistory = cache.get(main3File, repo, false); 511 assertEquals(7, updatedHistory.getHistoryEntries().size()); 512 assertSameEntries(histConstruct.getHistoryEntries(), 513 updatedHistory.getHistoryEntries(), false); 514 } 515 516 /** 517 * Make sure generating incremental history index in branched repository 518 * with renamed file produces correct history for the renamed file 519 * (i.e. there should not be history entries from the default branch made 520 * there after the branch was created). 521 */ 522 @EnabledOnOs({OS.LINUX, OS.MAC, OS.SOLARIS, OS.AIX, OS.OTHER}) 523 @EnabledForRepository(MERCURIAL) 524 @Test testRenamedFilePlusChangesBranched()525 void testRenamedFilePlusChangesBranched() throws Exception { 526 File reposRoot = new File(repositories.getSourceRoot(), "mercurial"); 527 History updatedHistory; 528 529 // The test expects support for renamed files. 530 env.setHandleHistoryOfRenamedFiles(true); 531 532 // Use tags for better coverage. 533 env.setTagsEnabled(true); 534 535 // Branch the repo and add one changeset. 536 runHgCommand(reposRoot, "unbundle", 537 Paths.get(getClass().getResource("/history/hg-branch.bundle").toURI()).toString()); 538 539 // Import changesets which rename one of the files in the default branch. 540 runHgCommand(reposRoot, "import", 541 Paths.get(getClass().getResource("/history/hg-export-renamed.txt").toURI()).toString()); 542 543 // Switch to the newly created branch. 544 runHgCommand(reposRoot, "update", "mybranch"); 545 546 // Generate history index. 547 // It is necessary to call getRepository() only after tags were enabled 548 // to produce list of tags. 549 Repository repo = RepositoryFactory.getRepository(reposRoot); 550 History historyToStore = repo.getHistory(reposRoot); 551 cache.store(historyToStore, repo); 552 553 // Import changesets which rename the file in the new branch. 554 runHgCommand(reposRoot, "import", 555 Paths.get(getClass().getResource("/history/hg-export-renamed-branched.txt").toURI()).toString()); 556 557 // Perform incremental reindex. 558 repo.createCache(cache, cache.getLatestCachedRevision(repo)); 559 560 // Check complete list of history entries for the renamed file. 561 File testFile = new File(reposRoot.toString() + File.separatorChar + "blog.txt"); 562 updatedHistory = cache.get(testFile, repo, false); 563 564 HistoryEntry e0 = new HistoryEntry( 565 "15:709c7a27f9fa", 566 new Date(1489160275L / 60 * 60 * 1000), // whole minutes only 567 "Vladimir Kotal <Vladimir.Kotal@oracle.com>", 568 "novels are so last century. Let's write a blog !", 569 true); 570 HistoryEntry e1 = new HistoryEntry( 571 "10:c4518ca0c841", 572 new Date(1415483555L / 60 * 60 * 1000), // whole minutes only 573 "Vladimir Kotal <Vladimir.Kotal@oracle.com>", 574 "branched", 575 true); 576 HistoryEntry e2 = new HistoryEntry( 577 "8:6a8c423f5624", 578 new Date(1362586899L / 60 * 60 * 1000), // whole minutes only 579 "Vladimir Kotal <vlada@devnull.cz>", 580 "first words of the novel", 581 true); 582 HistoryEntry e3 = new HistoryEntry( 583 "7:db1394c05268", 584 new Date(1362586862L / 60 * 60 * 1000), // whole minutes only 585 "Vladimir Kotal <vlada@devnull.cz>", 586 "book sounds too boring, let's do a novel !", 587 true); 588 HistoryEntry e4 = new HistoryEntry( 589 "6:e386b51ddbcc", 590 new Date(1362586839L / 60 * 60 * 1000), // whole minutes only 591 "Vladimir Kotal <vlada@devnull.cz>", 592 "stub of chapter 1", 593 true); 594 HistoryEntry e5 = new HistoryEntry( 595 "5:8706402863c6", 596 new Date(1362586805L / 60 * 60 * 1000), // whole minutes only 597 "Vladimir Kotal <vlada@devnull.cz>", 598 "I decided to actually start writing a book based on the first plaintext file.", 599 true); 600 HistoryEntry e6 = new HistoryEntry( 601 "4:e494d67af12f", 602 new Date(1362586747L / 60 * 60 * 1000), // whole minutes only 603 "Vladimir Kotal <vlada@devnull.cz>", 604 "first change", 605 true); 606 HistoryEntry e7 = new HistoryEntry( 607 "3:2058725c1470", 608 new Date(1362586483L / 60 * 60 * 1000), // whole minutes only 609 "Vladimir Kotal <vlada@devnull.cz>", 610 "initial checking of text files", 611 true); 612 613 History histConstruct = new History(); 614 LinkedList<HistoryEntry> entriesConstruct = new LinkedList<>(); 615 entriesConstruct.add(e0); 616 entriesConstruct.add(e1); 617 entriesConstruct.add(e2); 618 entriesConstruct.add(e3); 619 entriesConstruct.add(e4); 620 entriesConstruct.add(e5); 621 entriesConstruct.add(e6); 622 entriesConstruct.add(e7); 623 histConstruct.setHistoryEntries(entriesConstruct); 624 assertSameEntries(histConstruct.getHistoryEntries(), 625 updatedHistory.getHistoryEntries(), false); 626 } 627 628 /** 629 * Make sure produces correct history where several files are renamed in a single commit. 630 */ 631 @EnabledForRepository(SUBVERSION) 632 @Test testMultipleRenamedFiles()633 void testMultipleRenamedFiles() throws Exception { 634 createSvnRepository(); 635 636 File reposRoot = new File(repositories.getSourceRoot(), "subversion"); 637 History updatedHistory; 638 639 // The test expects support for renamed files. 640 env.setHandleHistoryOfRenamedFiles(true); 641 642 // Generate history index. 643 Repository repo = RepositoryFactory.getRepository(reposRoot); 644 History historyToStore = repo.getHistory(reposRoot); 645 cache.store(historyToStore, repo); 646 647 // Check complete list of history entries for the renamed file. 648 File testFile = new File(reposRoot.toString() + File.separatorChar + "FileZ.txt"); 649 updatedHistory = cache.get(testFile, repo, false); 650 assertEquals(3, updatedHistory.getHistoryEntries().size()); 651 652 HistoryEntry e0 = new HistoryEntry( 653 "10", 654 DateUtils.parseDate("2020-03-28T07:24:43.921Z", SVN_DATE_FORMAT), 655 "RichardH", 656 "Rename FileA to FileZ and FileB to FileX in a single commit", 657 true); 658 HistoryEntry e1 = new HistoryEntry( 659 "7", 660 DateUtils.parseDate("2020-03-28T07:21:55.273Z", SVN_DATE_FORMAT), 661 "RichardH", 662 "Amend file A", 663 true); 664 HistoryEntry e2 = new HistoryEntry( 665 "6", 666 DateUtils.parseDate("2020-03-28T07:21:05.888Z", SVN_DATE_FORMAT), 667 "RichardH", 668 "Add file A", 669 true); 670 671 History histConstruct = new History(); 672 LinkedList<HistoryEntry> entriesConstruct = new LinkedList<>(); 673 entriesConstruct.add(e0); 674 entriesConstruct.add(e1); 675 entriesConstruct.add(e2); 676 histConstruct.setHistoryEntries(entriesConstruct); 677 assertSameEntries(histConstruct.getHistoryEntries(), updatedHistory.getHistoryEntries(), false); 678 } 679 createSvnRepository()680 private void createSvnRepository() throws Exception { 681 var svnLog = FileHistoryCacheTest.class.getResource("/history/svnlog.dump"); 682 Path tempDir = Files.createTempDirectory("opengrok"); 683 Runtime.getRuntime().addShutdownHook(new Thread(() -> { 684 try { 685 IOUtils.removeRecursive(tempDir); 686 } catch (IOException e) { 687 // ignore 688 } 689 })); 690 String repo = tempDir.resolve("svn-repo").toString(); 691 var svnCreateRepoProcess = new ProcessBuilder("svnadmin", "create", repo).start(); 692 assertEquals(0, svnCreateRepoProcess.waitFor()); 693 694 var svnLoadRepoFromDumpProcess = new ProcessBuilder("svnadmin", "load", repo) 695 .redirectInput(Paths.get(svnLog.toURI()).toFile()) 696 .start(); 697 assertEquals(0, svnLoadRepoFromDumpProcess.waitFor()); 698 699 var svnCheckoutProcess = new ProcessBuilder("svn", "checkout", Path.of(repo).toUri().toString(), 700 Path.of(repositories.getSourceRoot()).resolve("subversion").toString()) 701 .start(); 702 assertEquals(0, svnCheckoutProcess.waitFor()); 703 } 704 changeFileAndCommit(Git git, File file, String comment)705 static void changeFileAndCommit(Git git, File file, String comment) throws Exception { 706 String authorName = "Foo Bar"; 707 String authorEmail = "foo@bar.com"; 708 709 try (FileOutputStream fos = new FileOutputStream(file, true)) { 710 fos.write(comment.getBytes(StandardCharsets.UTF_8)); 711 } 712 713 git.commit().setMessage(comment).setAuthor(authorName, authorEmail).setAll(true).call(); 714 } 715 716 /** 717 * Renamed files need special treatment when given repository supports per partes history retrieval. 718 * Specifically, when a file is detected as renamed, its history needs to be retrieved with upper bound, 719 * otherwise there would be duplicate history entries if there were subsequent changes to the file 720 * in the following history chunks. This test prevents that. 721 * @param maxCount maximum number of changesets to store in one go 722 * @throws Exception on error 723 */ 724 @ParameterizedTest 725 @ValueSource(ints = {2, 3, 4}) testRenamedFileHistoryWithPerPartes(int maxCount)726 void testRenamedFileHistoryWithPerPartes(int maxCount) throws Exception { 727 File repositoryRoot = new File(repositories.getSourceRoot(), "gitRenamedPerPartes"); 728 assertTrue(repositoryRoot.mkdir()); 729 File fooFile = new File(repositoryRoot, "foo.txt"); 730 if (!fooFile.createNewFile()) { 731 throw new IOException("Could not create file " + fooFile); 732 } 733 File barFile = new File(repositoryRoot, "bar.txt"); 734 735 // Looks like the file needs to have content of some length for the add/rm below 736 // to be detected as file rename by the [J]Git heuristics. 737 try (FileOutputStream fos = new FileOutputStream(fooFile)) { 738 fos.write("foo bar foo bar foo bar".getBytes(StandardCharsets.UTF_8)); 739 } 740 741 // Create a series of commits to one (renamed) file: 742 // - add the file 743 // - bunch of content commits to the file 744 // - rename the file 745 // - bunch of content commits to the renamed file 746 try (Git git = Git.init().setDirectory(repositoryRoot).call()) { 747 git.add().addFilepattern("foo.txt").call(); 748 changeFileAndCommit(git, fooFile, "initial"); 749 750 changeFileAndCommit(git, fooFile, "1"); 751 changeFileAndCommit(git, fooFile, "2"); 752 changeFileAndCommit(git, fooFile, "3"); 753 754 // git mv 755 assertTrue(fooFile.renameTo(barFile)); 756 git.add().addFilepattern("bar.txt").call(); 757 git.rm().addFilepattern("foo.txt").call(); 758 git.commit().setMessage("rename").setAuthor("foo", "foo@bar").setAll(true).call(); 759 760 changeFileAndCommit(git, barFile, "4"); 761 changeFileAndCommit(git, barFile, "5"); 762 changeFileAndCommit(git, barFile, "6"); 763 } 764 765 env.setHandleHistoryOfRenamedFiles(true); 766 Repository repository = RepositoryFactory.getRepository(repositoryRoot); 767 History history = repository.getHistory(repositoryRoot); 768 assertEquals(1, history.getRenamedFiles().size()); 769 GitRepository gitRepository = (GitRepository) repository; 770 GitRepository gitSpyRepository = Mockito.spy(gitRepository); 771 Mockito.when(gitSpyRepository.getPerPartesCount()).thenReturn(maxCount); 772 gitSpyRepository.createCache(cache, null); 773 774 assertTrue(barFile.isFile()); 775 History updatedHistory = cache.get(barFile, gitSpyRepository, false); 776 assertEquals(8, updatedHistory.getHistoryEntries().size()); 777 } 778 779 /** 780 * Make sure produces correct history for a renamed and moved file in Subversion. 781 */ 782 @EnabledForRepository(SUBVERSION) 783 @Test testRenamedFile()784 void testRenamedFile() throws Exception { 785 createSvnRepository(); 786 787 File reposRoot = new File(repositories.getSourceRoot(), "subversion"); 788 History updatedHistory; 789 790 // The test expects support for renamed files. 791 env.setHandleHistoryOfRenamedFiles(true); 792 793 // Generate history index. 794 Repository repo = RepositoryFactory.getRepository(reposRoot); 795 History historyToStore = repo.getHistory(reposRoot); 796 cache.store(historyToStore, repo); 797 798 // Check complete list of history entries for the renamed file. 799 File testFile = new File(reposRoot.toString() + File.separatorChar 800 + "subfolder" + File.separatorChar + "TestFileRenamedAgain.txt"); 801 updatedHistory = cache.get(testFile, repo, false); 802 assertEquals(4, updatedHistory.getHistoryEntries().size()); 803 804 HistoryEntry e0 = new HistoryEntry( 805 "5", 806 DateUtils.parseDate("2020-03-28T07:20:11.821Z", SVN_DATE_FORMAT), 807 "RichardH", 808 "Moved file to subfolder and renamed", 809 true); 810 HistoryEntry e1 = new HistoryEntry( 811 "3", 812 DateUtils.parseDate("2020-03-28T07:19:03.145Z", SVN_DATE_FORMAT), 813 "RichardH", 814 "Edited content", 815 true); 816 HistoryEntry e2 = new HistoryEntry( 817 "2", 818 DateUtils.parseDate("2020-03-28T07:18:29.481Z", SVN_DATE_FORMAT), 819 "RichardH", 820 "Rename file", 821 true); 822 HistoryEntry e3 = new HistoryEntry( 823 "1", 824 DateUtils.parseDate("2020-03-28T07:17:54.756Z", SVN_DATE_FORMAT), 825 "RichardH", 826 "Add initial file", 827 true); 828 829 History histConstruct = new History(); 830 LinkedList<HistoryEntry> entriesConstruct = new LinkedList<>(); 831 entriesConstruct.add(e0); 832 entriesConstruct.add(e1); 833 entriesConstruct.add(e2); 834 entriesConstruct.add(e3); 835 histConstruct.setHistoryEntries(entriesConstruct); 836 assertSameEntries(histConstruct.getHistoryEntries(), 837 updatedHistory.getHistoryEntries(), false); 838 } 839 checkNoHistoryFetchRepo(String repoName, String filename, boolean hasHistory)840 private void checkNoHistoryFetchRepo(String repoName, String filename, boolean hasHistory) throws Exception { 841 842 File reposRoot = new File(repositories.getSourceRoot(), repoName); 843 844 // Make sure the file exists in the repository. 845 File repoFile = new File(reposRoot, filename); 846 assertTrue(repoFile.exists()); 847 848 // Try to fetch the history for given file. With default setting of 849 // FetchHistoryWhenNotInCache this should get the history even if not in cache. 850 History retrievedHistory = HistoryGuru.getInstance().getHistory(repoFile, true); 851 assertEquals(hasHistory, retrievedHistory != null); 852 } 853 854 /* 855 * Functional test for the FetchHistoryWhenNotInCache configuration option. 856 */ 857 @EnabledForRepository({MERCURIAL, SCCS}) 858 @Test testNoHistoryFetch()859 void testNoHistoryFetch() throws Exception { 860 env.setFetchHistoryWhenNotInCache(false); 861 862 // Pretend we are done with first phase of indexing. 863 env.setIndexer(true); 864 HistoryGuru.getInstance().setHistoryIndexDone(); 865 866 // First try repo with ability to fetch history for directories. 867 checkNoHistoryFetchRepo("mercurial", "main.c", false); 868 // Second try repo which can fetch history of individual files only. 869 checkNoHistoryFetchRepo("teamware", "header.h", true); 870 } 871 872 /** 873 * Test history when activating PathAccepter for ignoring files. 874 */ 875 @EnabledOnOs({OS.LINUX, OS.MAC, OS.SOLARIS, OS.AIX, OS.OTHER}) 876 @EnabledForRepository(MERCURIAL) 877 @Test testStoreAndTryToGetIgnored()878 void testStoreAndTryToGetIgnored() throws Exception { 879 env.getIgnoredNames().add("f:Make*"); 880 881 File reposRoot = new File(repositories.getSourceRoot(), "mercurial"); 882 883 Repository repo = RepositoryFactory.getRepository(reposRoot); 884 History historyToStore = repo.getHistory(reposRoot); 885 886 cache.store(historyToStore, repo); 887 888 // test reindex history 889 History historyNull = new History(); 890 cache.store(historyNull, repo); 891 892 // test get history for single file 893 File makefile = new File(reposRoot, "Makefile"); 894 assertTrue(makefile.exists(), "" + makefile + " should exist"); 895 896 History retrievedHistory = cache.get(makefile, repo, true); 897 assertNull(retrievedHistory, "history for Makefile should be null"); 898 899 // Gross that we can break encapsulation, but oh well. 900 env.getIgnoredNames().clear(); 901 // Need to refresh the history since it was changed in the cache.store(). 902 historyToStore = repo.getHistory(reposRoot); 903 cache.store(historyToStore, repo); 904 retrievedHistory = cache.get(makefile, repo, true); 905 assertNotNull(retrievedHistory, "history for Makefile should not be null"); 906 } 907 908 @ParameterizedTest 909 @ValueSource(strings = { 910 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + 911 "<java version=\"11.0.8\" class=\"java.beans.XMLDecoder\">\n" + 912 " <object class=\"java.lang.Runtime\" method=\"getRuntime\">\n" + 913 " <void method=\"exec\">\n" + 914 " <array class=\"java.lang.String\" length=\"2\">\n" + 915 " <void index=\"0\">\n" + 916 " <string>/usr/bin/nc</string>\n" + 917 " </void>\n" + 918 " <void index=\"1\">\n" + 919 " <string>-l</string>\n" + 920 " </void>\n" + 921 " </array>\n" + 922 " </void>\n" + 923 " </object>\n" + 924 "</java>", 925 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + 926 "<java version=\"11.0.8\" class=\"java.beans.XMLDecoder\">\n" + 927 " <object class=\"java.lang.ProcessBuilder\">\n" + 928 " <array class=\"java.lang.String\" length=\"1\" >\n" + 929 " <void index=\"0\"> \n" + 930 " <string>/usr/bin/curl https://oracle.com</string>\n" + 931 " </void>\n" + 932 " </array>\n" + 933 " <void method=\"start\"/>\n" + 934 " </object>\n" + 935 "</java>", 936 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + 937 "<java version=\"11.0.8\" class=\"java.beans.XMLDecoder\">\n" + 938 " <object class = \"java.io.FileOutputStream\"> \n" + 939 " <string>opengrok_test.txt</string>\n" + 940 " <method name = \"write\">\n" + 941 " <array class=\"byte\" length=\"3\">\n" + 942 " <void index=\"0\"><byte>96</byte></void>\n" + 943 " <void index=\"1\"><byte>96</byte></void>\n" + 944 " <void index=\"2\"><byte>96</byte></void>\n" + 945 " </array>\n" + 946 " </method>\n" + 947 " <method name=\"close\"/>\n" + 948 " </object>\n" + 949 "</java>" 950 }) testDeserializationOfNotWhiteListedClassThrowsError(final String exploit)951 void testDeserializationOfNotWhiteListedClassThrowsError(final String exploit) { 952 assertThrows(IllegalAccessError.class, () -> FileHistoryCache.readCache(exploit)); 953 } 954 955 @Test testReadCacheValid()956 void testReadCacheValid() throws IOException { 957 File testFile = new File(FileHistoryCacheTest.class.getClassLoader(). 958 getResource("history/FileHistoryCache.java.gz").getFile()); 959 History history = FileHistoryCache.readCache(testFile); 960 assertNotNull(history); 961 assertEquals(30, history.getHistoryEntries().size()); 962 } 963 } 964