xref: /OpenGrok/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java (revision 455d146621865162187c700c0dcfc66d475a5106)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * See LICENSE.txt included in this distribution for the specific
9  * language governing permissions and limitations under the License.
10  *
11  * When distributing Covered Code, include this CDDL HEADER in each
12  * file and include the License file at LICENSE.txt.
13  * If applicable, add the following below this CDDL HEADER, with the
14  * fields enclosed by brackets "[]" replaced with your own identifying
15  * information: Portions Copyright [yyyy] [name of copyright owner]
16  *
17  * CDDL HEADER END
18  */
19 
20 /*
21  * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
22  * Portions Copyright (c) 2018, 2020, Chris Fraire <cfraire@me.com>.
23  */
24 package org.opengrok.indexer.history;
25 
26 import java.beans.Encoder;
27 import java.beans.Expression;
28 import java.beans.PersistenceDelegate;
29 import java.beans.XMLDecoder;
30 import java.beans.XMLEncoder;
31 import java.io.BufferedInputStream;
32 import java.io.BufferedOutputStream;
33 import java.io.BufferedReader;
34 import java.io.BufferedWriter;
35 import java.io.ByteArrayInputStream;
36 import java.io.File;
37 import java.io.FileInputStream;
38 import java.io.FileOutputStream;
39 import java.io.FileReader;
40 import java.io.IOException;
41 import java.io.InputStream;
42 import java.io.OutputStreamWriter;
43 import java.io.Writer;
44 import java.nio.file.Files;
45 import java.nio.file.NoSuchFileException;
46 import java.nio.file.Path;
47 import java.nio.file.Paths;
48 import java.util.ArrayList;
49 import java.util.Collections;
50 import java.util.Date;
51 import java.util.HashMap;
52 import java.util.List;
53 import java.util.ListIterator;
54 import java.util.Map;
55 import java.util.Set;
56 import java.util.concurrent.CountDownLatch;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.logging.Level;
59 import java.util.logging.Logger;
60 import java.util.stream.Collectors;
61 import java.util.zip.GZIPInputStream;
62 import java.util.zip.GZIPOutputStream;
63 
64 import io.micrometer.core.instrument.Counter;
65 import io.micrometer.core.instrument.MeterRegistry;
66 import org.jetbrains.annotations.Nullable;
67 import org.jetbrains.annotations.TestOnly;
68 import org.jetbrains.annotations.VisibleForTesting;
69 import org.opengrok.indexer.Metrics;
70 import org.opengrok.indexer.configuration.PathAccepter;
71 import org.opengrok.indexer.configuration.RuntimeEnvironment;
72 import org.opengrok.indexer.logger.LoggerFactory;
73 import org.opengrok.indexer.util.ForbiddenSymlinkException;
74 import org.opengrok.indexer.util.IOUtils;
75 import org.opengrok.indexer.util.Progress;
76 import org.opengrok.indexer.util.Statistics;
77 import org.opengrok.indexer.util.TandemPath;
78 
79 /**
80  * Class representing file based storage of per source file history.
81  */
82 class FileHistoryCache implements HistoryCache {
83 
84     private static final Logger LOGGER = LoggerFactory.getLogger(FileHistoryCache.class);
85     private static final RuntimeEnvironment env = RuntimeEnvironment.getInstance();
86 
87     private final Object lock = new Object();
88 
89     private static final String HISTORY_CACHE_DIR_NAME = "historycache";
90     private static final String LATEST_REV_FILE_NAME = "OpenGroklatestRev";
91 
92     private final PathAccepter pathAccepter = env.getPathAccepter();
93 
94     private Counter fileHistoryCacheHits;
95     private Counter fileHistoryCacheMisses;
96 
97     /**
98      * Generate history cache for single renamed file.
99      * @param filename file path
100      * @param repository repository
101      * @param root root
102      * @param tillRevision end revision (can be null)
103      */
doRenamedFileHistory(String filename, File file, Repository repository, File root, String tillRevision)104     public void doRenamedFileHistory(String filename, File file, Repository repository, File root, String tillRevision)
105             throws HistoryException {
106 
107         History history;
108 
109         if (tillRevision != null) {
110             if (!(repository instanceof RepositoryWithPerPartesHistory)) {
111                 throw new RuntimeException("cannot use non null tillRevision on repository");
112             }
113 
114             RepositoryWithPerPartesHistory repo = (RepositoryWithPerPartesHistory) repository;
115             history = repo.getHistory(file, null, tillRevision);
116         } else {
117             history = repository.getHistory(file);
118         }
119 
120         history.strip();
121         doFileHistory(filename, history, repository, root, true);
122     }
123 
124     /**
125      * Generate history cache for single file.
126      * @param filename name of the file
127      * @param history history
128      * @param repository repository object in which the file belongs
129      * @param root root of the source repository
130      * @param renamed true if the file was renamed in the past
131      */
doFileHistory(String filename, History history, Repository repository, File root, boolean renamed)132     private void doFileHistory(String filename, History history, Repository repository, File root, boolean renamed)
133             throws HistoryException {
134 
135         File file = new File(root, filename);
136         if (file.isDirectory()) {
137             return;
138         }
139 
140         // Assign tags to changesets they represent.
141         if (env.isTagsEnabled() && repository.hasFileBasedTags()) {
142             repository.assignTagsInHistory(history);
143         }
144 
145         storeFile(history, file, repository, !renamed);
146     }
147 
148     static class FilePersistenceDelegate extends PersistenceDelegate {
149         @Override
instantiate(Object oldInstance, Encoder out)150         protected Expression instantiate(Object oldInstance, Encoder out) {
151             File f = (File) oldInstance;
152             return new Expression(oldInstance, f.getClass(), "new",
153                 new Object[] {f.toString()});
154         }
155     }
156 
157     @Override
initialize()158     public void initialize() {
159         MeterRegistry meterRegistry = Metrics.getRegistry();
160         if (meterRegistry != null) {
161             fileHistoryCacheHits = Counter.builder("filehistorycache.history.get").
162                     description("file history cache hits").
163                     tag("what", "hits").
164                     register(meterRegistry);
165             fileHistoryCacheMisses = Counter.builder("filehistorycache.history.get").
166                     description("file history cache misses").
167                     tag("what", "miss").
168                     register(meterRegistry);
169         }
170     }
171 
getFileHistoryCacheHits()172     double getFileHistoryCacheHits() {
173         return fileHistoryCacheHits.count();
174     }
175 
176     @Override
optimize()177     public void optimize() {
178         // nothing to do
179     }
180 
181     @Override
supportsRepository(Repository repository)182     public boolean supportsRepository(Repository repository) {
183         // all repositories are supported
184         return true;
185     }
186 
187     /**
188      * Get a <code>File</code> object describing the cache file.
189      *
190      * @param file the file to find the cache for
191      * @return file that might contain cached history for <code>file</code>
192      */
getCachedFile(File file)193     private static File getCachedFile(File file) throws HistoryException, ForbiddenSymlinkException {
194 
195         StringBuilder sb = new StringBuilder();
196         sb.append(env.getDataRootPath());
197         sb.append(File.separatorChar);
198         sb.append(HISTORY_CACHE_DIR_NAME);
199 
200         try {
201             String add = env.getPathRelativeToSourceRoot(file);
202             if (add.length() == 0) {
203                 add = File.separator;
204             }
205             sb.append(add);
206         } catch (IOException e) {
207             throw new HistoryException("Failed to get path relative to source root for " + file, e);
208         }
209 
210         return new File(TandemPath.join(sb.toString(), ".gz"));
211     }
212 
getDecoder(InputStream in)213     private static XMLDecoder getDecoder(InputStream in) {
214         return new XMLDecoder(in, null, null, new HistoryClassLoader());
215     }
216 
217     @TestOnly
readCache(String xmlconfig)218     static History readCache(String xmlconfig) {
219         final ByteArrayInputStream in = new ByteArrayInputStream(xmlconfig.getBytes());
220         try (XMLDecoder d = getDecoder(in)) {
221             return (History) d.readObject();
222         }
223     }
224 
225     /**
226      * Read history from a file.
227      */
readCache(File file)228     static History readCache(File file) throws IOException {
229         try (FileInputStream in = new FileInputStream(file);
230             XMLDecoder d = getDecoder(new GZIPInputStream(new BufferedInputStream(in)))) {
231             return (History) d.readObject();
232         }
233     }
234 
235     /**
236      * Store history in file on disk.
237      * @param dir directory where the file will be saved
238      * @param history history to store
239      * @param cacheFile the file to store the history to
240      */
writeHistoryToFile(File dir, History history, File cacheFile)241     private void writeHistoryToFile(File dir, History history, File cacheFile) throws HistoryException {
242 
243         if (LOGGER.isLoggable(Level.FINEST)) {
244             LOGGER.log(Level.FINEST, "writing history entries to ''{0}'': {1}",
245                     new Object[]{cacheFile, history.getRevisionList()});
246         }
247 
248         // We have a problem that multiple threads may access the cache layer
249         // at the same time. Since I would like to avoid read-locking, I just
250         // serialize the write access to the cache file. The generation of the
251         // cache file would most likely be executed during index generation, and
252         // that happens sequential anyway....
253         // Generate the file with a temporary name and move it into place when
254         // done, so it is not necessary to protect the readers for partially updated
255         // files...
256         final File output;
257         try {
258             output = File.createTempFile("oghist", null, dir);
259             try (FileOutputStream out = new FileOutputStream(output);
260                 XMLEncoder e = new XMLEncoder(new GZIPOutputStream(
261                     new BufferedOutputStream(out)))) {
262                 e.setPersistenceDelegate(File.class,
263                         new FilePersistenceDelegate());
264                 e.writeObject(history);
265             }
266         } catch (IOException ioe) {
267             throw new HistoryException("Failed to write history", ioe);
268         }
269         synchronized (lock) {
270             if (!cacheFile.delete() && cacheFile.exists()) {
271                 if (!output.delete()) {
272                     LOGGER.log(Level.WARNING,
273                             "Failed to remove temporary history cache file");
274                 }
275                 throw new HistoryException(String.format("Cache file '%s' exists, and could not be deleted.",
276                         cacheFile));
277             }
278             if (!output.renameTo(cacheFile)) {
279                 try {
280                     Files.delete(output.toPath());
281                 } catch (IOException e) {
282                     throw new HistoryException("failed to delete output file", e);
283                 }
284                 throw new HistoryException(String.format("Failed to rename cache temporary file '%s' to '%s'",
285                         output, cacheFile));
286             }
287         }
288     }
289 
290     /**
291      * Read history from cacheFile and merge it with histNew, return merged history.
292      *
293      * @param cacheFile file to where the history object will be stored
294      * @param histNew history object with new history entries
295      * @param repo repository to where pre pre-image of the cacheFile belong
296      * @return merged history (can be null if merge failed for some reason)
297      */
mergeOldAndNewHistory(File cacheFile, History histNew, Repository repo)298     private History mergeOldAndNewHistory(File cacheFile, History histNew, Repository repo) {
299 
300         History histOld;
301         History history = null;
302 
303         try {
304             histOld = readCache(cacheFile);
305             // Merge old history with the new history.
306             List<HistoryEntry> listOld = histOld.getHistoryEntries();
307             if (!listOld.isEmpty()) {
308                 if (LOGGER.isLoggable(Level.FINEST)) {
309                     LOGGER.log(Level.FINEST, "for ''{0}'' merging old history {1} with new history {2}",
310                             new Object[]{cacheFile, histOld.getRevisionList(), histNew.getRevisionList()});
311                 }
312                 List<HistoryEntry> listNew = histNew.getHistoryEntries();
313                 ListIterator<HistoryEntry> li = listNew.listIterator(listNew.size());
314                 while (li.hasPrevious()) {
315                     listOld.add(0, li.previous());
316                 }
317                 history = new History(listOld);
318 
319                 // Re-tag the changesets in case there have been some new
320                 // tags added to the repository. Technically we should just
321                 // re-tag the last revision from the listOld however this
322                 // does not solve the problem when listNew contains new tags
323                 // retroactively tagging changesets from listOld, so we resort
324                 // to this somewhat crude solution of re-tagging from scratch.
325                 if (env.isTagsEnabled() && repo.hasFileBasedTags()) {
326                     history.strip();
327                     repo.assignTagsInHistory(history);
328                 }
329             }
330         } catch (IOException ex) {
331             LOGGER.log(Level.SEVERE,
332                 String.format("Cannot open history cache file %s", cacheFile.getPath()), ex);
333         }
334 
335         return history;
336     }
337 
338     /**
339      * Store history object (encoded as XML and compressed with gzip) in a file.
340      *
341      * @param histNew history object to store
342      * @param file file to store the history object into
343      * @param repo repository for the file
344      * @param mergeHistory whether to merge the history with existing or
345      *                     store the histNew as is
346      */
storeFile(History histNew, File file, Repository repo, boolean mergeHistory)347     private void storeFile(History histNew, File file, Repository repo, boolean mergeHistory) throws HistoryException {
348         File cacheFile;
349         try {
350             cacheFile = getCachedFile(file);
351         } catch (ForbiddenSymlinkException e) {
352             LOGGER.log(Level.FINER, e.getMessage());
353             return;
354         }
355         History history = histNew;
356 
357         File dir = cacheFile.getParentFile();
358         // calling isDirectory twice to prevent a race condition
359         if (!dir.isDirectory() && !dir.mkdirs() && !dir.isDirectory()) {
360             throw new HistoryException("Unable to create cache directory '" + dir + "'.");
361         }
362 
363         if (mergeHistory && cacheFile.exists()) {
364             history = mergeOldAndNewHistory(cacheFile, histNew, repo);
365         }
366 
367         // If the merge failed, null history will be returned.
368         // In such case store at least new history as the best effort.
369         if (history == null) {
370             LOGGER.log(Level.WARNING, "history cache for file ''{0}'' truncated to new history", file);
371             history = histNew;
372         }
373 
374         writeHistoryToFile(dir, history, cacheFile);
375     }
376 
storeFile(History histNew, File file, Repository repo)377     private void storeFile(History histNew, File file, Repository repo) throws HistoryException {
378         storeFile(histNew, file, repo, false);
379     }
380 
finishStore(Repository repository, String latestRev)381     private void finishStore(Repository repository, String latestRev) {
382         String histDir = getRepositoryHistDataDirname(repository);
383         if (histDir == null || !(new File(histDir)).isDirectory()) {
384             // If the history was not created for some reason (e.g. temporary
385             // failure), do not create the CachedRevision file as this would
386             // create confusion (once it starts working again).
387             LOGGER.log(Level.WARNING,
388                 "Could not store history for repository {0}: {1} is not a directory",
389                 new Object[]{repository, histDir});
390         } else {
391             storeLatestCachedRevision(repository, latestRev);
392         }
393     }
394 
395     @Override
store(History history, Repository repository)396     public void store(History history, Repository repository) throws HistoryException {
397         store(history, repository, null);
398     }
399 
createFileMap(History history, HashMap<String, List<HistoryEntry>> map)400     private String createFileMap(History history, HashMap<String, List<HistoryEntry>> map) {
401         String latestRev = null;
402         HashMap<String, Boolean> acceptanceCache = new HashMap<>();
403 
404         /*
405          * Go through all history entries for this repository (acquired through
406          * history/log command executed for top-level directory of the repo
407          * and parsed into HistoryEntry structures) and create hash map which
408          * maps file names into list of HistoryEntry structures corresponding
409          * to changesets in which the file was modified.
410          */
411         for (HistoryEntry e : history.getHistoryEntries()) {
412             // The history entries are sorted from newest to oldest.
413             if (latestRev == null) {
414                 latestRev = e.getRevision();
415             }
416             for (String s : e.getFiles()) {
417                 /*
418                  * We do not want to generate history cache for files which
419                  * do not currently exist in the repository.
420                  *
421                  * Also we cache the result of this evaluation to boost
422                  * performance, since a particular file can appear in many
423                  * repository revisions.
424                  */
425                 File test = new File(env.getSourceRootPath() + s);
426                 String testKey = test.getAbsolutePath();
427                 Boolean cachedAcceptance = acceptanceCache.get(testKey);
428                 if (cachedAcceptance != null) {
429                     if (!cachedAcceptance) {
430                         continue;
431                     }
432                 } else {
433                     boolean testResult = test.exists() && pathAccepter.accept(test);
434                     acceptanceCache.put(testKey, testResult);
435                     if (!testResult) {
436                         continue;
437                     }
438                 }
439 
440                 List<HistoryEntry> list = map.computeIfAbsent(s, k -> new ArrayList<>());
441 
442                 list.add(e);
443             }
444         }
445         return latestRev;
446     }
447 
getRevisionString(String revision)448     private static String getRevisionString(String revision) {
449         if (revision == null) {
450             return "end of history";
451         } else {
452             return "revision " + revision;
453         }
454     }
455 
456     /**
457      * Store history for the whole repository in directory hierarchy resembling
458      * the original repository structure. History of individual files will be
459      * stored under this hierarchy, each file containing history of
460      * corresponding source file.
461      *
462      * <p>
463      * <b>Note that the history object will be changed in the process of storing the history into cache.
464      * Namely the list of files from the history entries will be stripped.</b>
465      * </p>
466      *
467      * @param history history object to process into per-file histories
468      * @param repository repository object
469      * @param tillRevision end revision (can be null)
470      */
471     @Override
store(History history, Repository repository, String tillRevision)472     public void store(History history, Repository repository, String tillRevision) throws HistoryException {
473 
474         final boolean handleRenamedFiles = repository.isHandleRenamedFiles();
475 
476         String latestRev = null;
477 
478         // Return immediately when there is nothing to do.
479         List<HistoryEntry> entries = history.getHistoryEntries();
480         if (entries.isEmpty()) {
481             return;
482         }
483 
484         HashMap<String, List<HistoryEntry>> map = new HashMap<>();
485         latestRev = createFileMap(history, map);
486 
487         // File based history cache does not store files for individual changesets so strip them.
488         history.strip();
489 
490         File histDataDir = new File(getRepositoryHistDataDirname(repository));
491         // Check the directory again in case of races (might happen in the presence of sub-repositories).
492         if (!histDataDir.isDirectory() && !histDataDir.mkdirs() && !histDataDir.isDirectory()) {
493             LOGGER.log(Level.WARNING, "cannot create history cache directory for ''{0}''", histDataDir);
494         }
495 
496         Set<String> regularFiles = map.keySet().stream().
497                 filter(e -> !history.isRenamed(e)).collect(Collectors.toSet());
498         createDirectoriesForFiles(regularFiles, repository, "regular files for history till " +
499                 getRevisionString(tillRevision));
500 
501         /*
502          * Now traverse the list of files from the hash map built above and for each file store its history
503          * (saved in the value of the hash map entry for the file) in a file.
504          * The renamed files will be handled separately.
505          */
506         LOGGER.log(Level.FINE, "Storing history for {0} regular files in repository ''{1}'' till {2}",
507                 new Object[]{regularFiles.size(), repository, getRevisionString(tillRevision)});
508         final File root = env.getSourceRootFile();
509 
510         final CountDownLatch latch = new CountDownLatch(regularFiles.size());
511         AtomicInteger fileHistoryCount = new AtomicInteger();
512         try (Progress progress = new Progress(LOGGER,
513                 String.format("history cache for regular files of %s till %s", repository,
514                         getRevisionString(tillRevision)),
515                 regularFiles.size())) {
516             for (String file : regularFiles) {
517                 env.getIndexerParallelizer().getHistoryFileExecutor().submit(() -> {
518                     try {
519                         doFileHistory(file, new History(map.get(file)), repository, root, false);
520                         fileHistoryCount.getAndIncrement();
521                     } catch (Exception ex) {
522                         // We want to catch any exception since we are in thread.
523                         LOGGER.log(Level.WARNING, "doFileHistory() got exception ", ex);
524                     } finally {
525                         latch.countDown();
526                         progress.increment();
527                     }
528                 });
529             }
530 
531             // Wait for the executors to finish.
532             try {
533                 latch.await();
534             } catch (InterruptedException ex) {
535                 LOGGER.log(Level.SEVERE, "latch exception", ex);
536             }
537             LOGGER.log(Level.FINE, "Stored history for {0} regular files in repository ''{1}''",
538                     new Object[]{fileHistoryCount, repository});
539         }
540 
541         if (!handleRenamedFiles) {
542             finishStore(repository, latestRev);
543             return;
544         }
545 
546         storeRenamed(history.getRenamedFiles(), repository, tillRevision);
547 
548         finishStore(repository, latestRev);
549     }
550 
551     /**
552      * handle renamed files (in parallel).
553      * @param renamedFiles set of renamed file paths
554      * @param repository repository
555      * @param tillRevision end revision (can be null)
556      */
storeRenamed(Set<String> renamedFiles, Repository repository, String tillRevision)557     public void storeRenamed(Set<String> renamedFiles, Repository repository, String tillRevision) throws HistoryException {
558         final File root = env.getSourceRootFile();
559         if (renamedFiles.isEmpty()) {
560             return;
561         }
562 
563         renamedFiles = renamedFiles.stream().filter(f -> new File(env.getSourceRootPath() + f).exists()).
564                 collect(Collectors.toSet());
565         LOGGER.log(Level.FINE, "Storing history for {0} renamed files in repository ''{1}'' till {2}",
566                 new Object[]{renamedFiles.size(), repository, getRevisionString(tillRevision)});
567 
568         createDirectoriesForFiles(renamedFiles, repository, "renamed files for history " +
569                 getRevisionString(tillRevision));
570 
571         final Repository repositoryF = repository;
572         final CountDownLatch latch = new CountDownLatch(renamedFiles.size());
573         AtomicInteger renamedFileHistoryCount = new AtomicInteger();
574         try (Progress progress = new Progress(LOGGER,
575                 String.format("history cache for renamed files of %s till %s", repository,
576                         getRevisionString(tillRevision)),
577                 renamedFiles.size())) {
578             for (final String file : renamedFiles) {
579                 env.getIndexerParallelizer().getHistoryFileExecutor().submit(() -> {
580                     try {
581                         doRenamedFileHistory(file,
582                                 new File(env.getSourceRootPath() + file),
583                                 repositoryF, root, tillRevision);
584                         renamedFileHistoryCount.getAndIncrement();
585                     } catch (Exception ex) {
586                         // We want to catch any exception since we are in thread.
587                         LOGGER.log(Level.WARNING, "doFileHistory() got exception ", ex);
588                     } finally {
589                         latch.countDown();
590                         progress.increment();
591                     }
592                 });
593             }
594 
595             // Wait for the executors to finish.
596             try {
597                 // Wait for the executors to finish.
598                 latch.await();
599             } catch (InterruptedException ex) {
600                 LOGGER.log(Level.SEVERE, "latch exception", ex);
601             }
602         }
603         LOGGER.log(Level.FINE, "Stored history for {0} renamed files in repository ''{1}''",
604                 new Object[]{renamedFileHistoryCount.intValue(), repository});
605     }
606 
createDirectoriesForFiles(Set<String> files, Repository repository, String label)607     private void createDirectoriesForFiles(Set<String> files, Repository repository, String label)
608             throws HistoryException {
609 
610         // The directories for the files have to be created before
611         // the actual files otherwise storeFile() might be racing for
612         // mkdirs() if there are multiple files from single directory
613         // handled in parallel.
614         Statistics elapsed = new Statistics();
615         LOGGER.log(Level.FINE, "Starting directory creation for {0} ({1}): {2} directories",
616                 new Object[]{repository, label, files.size()});
617         for (final String file : files) {
618             File cache;
619             try {
620                 cache = getCachedFile(new File(env.getSourceRootPath() + file));
621             } catch (ForbiddenSymlinkException ex) {
622                 LOGGER.log(Level.FINER, ex.getMessage());
623                 continue;
624             }
625             File dir = cache.getParentFile();
626 
627             if (!dir.isDirectory() && !dir.mkdirs()) {
628                 LOGGER.log(Level.WARNING, "Unable to create cache directory ''{0}''.", dir);
629             }
630         }
631         elapsed.report(LOGGER, String.format("Done creating directories for %s (%s)", repository, label));
632     }
633 
634     @Override
get(File file, Repository repository, boolean withFiles)635     public History get(File file, Repository repository, boolean withFiles)
636             throws HistoryException, ForbiddenSymlinkException {
637 
638         File cacheFile = getCachedFile(file);
639         if (isUpToDate(file, cacheFile)) {
640             try {
641                 if (fileHistoryCacheHits != null) {
642                     fileHistoryCacheHits.increment();
643                 }
644                 return readCache(cacheFile);
645             } catch (Exception e) {
646                 LOGGER.log(Level.WARNING, String.format("Error when reading cache file '%s'", cacheFile), e);
647             }
648         }
649 
650         if (fileHistoryCacheMisses != null) {
651             fileHistoryCacheMisses.increment();
652         }
653 
654         return null;
655     }
656 
657     /**
658      * @param file the file to check
659      * @param cachedFile the file which contains the cached history for the file
660      * @return {@code true} if the cache is up-to-date for the file, {@code false} otherwise
661      */
isUpToDate(File file, File cachedFile)662     private boolean isUpToDate(File file, File cachedFile) {
663         return cachedFile != null && cachedFile.exists() && file.lastModified() <= cachedFile.lastModified();
664     }
665 
666     @Override
hasCacheForFile(File file)667     public boolean hasCacheForFile(File file) throws HistoryException {
668         try {
669             return getCachedFile(file).exists();
670         } catch (ForbiddenSymlinkException ex) {
671             LOGGER.log(Level.FINER, ex.getMessage());
672             return false;
673         }
674     }
675 
676     @VisibleForTesting
getRepositoryHistDataDirname(Repository repository)677     public static String getRepositoryHistDataDirname(Repository repository) {
678         String repoDirBasename;
679 
680         try {
681             repoDirBasename = env.getPathRelativeToSourceRoot(new File(repository.getDirectoryName()));
682         } catch (IOException ex) {
683             LOGGER.log(Level.WARNING,
684                     String.format("Could not resolve repository %s relative to source root", repository), ex);
685             return null;
686         } catch (ForbiddenSymlinkException ex) {
687             LOGGER.log(Level.FINER, ex.getMessage());
688             return null;
689         }
690 
691         return env.getDataRootPath() + File.separatorChar
692             + FileHistoryCache.HISTORY_CACHE_DIR_NAME
693             + repoDirBasename;
694     }
695 
getRepositoryCachedRevPath(Repository repository)696     private String getRepositoryCachedRevPath(Repository repository) {
697         String histDir = getRepositoryHistDataDirname(repository);
698         if (histDir == null) {
699             return null;
700         }
701         return histDir + File.separatorChar + LATEST_REV_FILE_NAME;
702     }
703 
704     /**
705      * Store latest indexed revision for the repository under data directory.
706      * @param repository repository
707      * @param rev latest revision which has been just indexed
708      */
storeLatestCachedRevision(Repository repository, String rev)709     private void storeLatestCachedRevision(Repository repository, String rev) {
710         Path newPath = Path.of(getRepositoryCachedRevPath(repository));
711         try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(newPath.toFile())))) {
712             writer.write(rev);
713         } catch (IOException ex) {
714             LOGGER.log(Level.WARNING,
715                     String.format("Cannot write latest cached revision to file for repository %s", repository), ex);
716         }
717     }
718 
719     @Override
720     @Nullable
getLatestCachedRevision(Repository repository)721     public String getLatestCachedRevision(Repository repository) {
722         return getCachedRevision(repository, getRepositoryCachedRevPath(repository));
723     }
724 
725     @Nullable
getCachedRevision(Repository repository, String revPath)726     private String getCachedRevision(Repository repository, String revPath) {
727         String rev;
728         BufferedReader input;
729 
730         if (revPath == null) {
731             LOGGER.log(Level.WARNING, "no rev path for repository {0}", repository);
732             return null;
733         }
734 
735         try {
736             input = new BufferedReader(new FileReader(revPath));
737             try {
738                 rev = input.readLine();
739             } catch (java.io.IOException e) {
740                 LOGGER.log(Level.WARNING, "failed to load ", e);
741                 return null;
742             } finally {
743                 try {
744                     input.close();
745                 } catch (java.io.IOException e) {
746                     LOGGER.log(Level.WARNING, "failed to close", e);
747                 }
748             }
749         } catch (java.io.FileNotFoundException e) {
750             LOGGER.log(Level.FINE,
751                 "not loading latest cached revision file from {0}", revPath);
752             return null;
753         }
754 
755         return rev;
756     }
757 
758     @Override
getLastModifiedTimes( File directory, Repository repository)759     public Map<String, Date> getLastModifiedTimes(
760             File directory, Repository repository) {
761         // We don't have a good way to get this information from the file
762         // cache, so leave it to the caller to find a reasonable time to
763         // display (typically the last modified time on the file system).
764         return Collections.emptyMap();
765     }
766 
767     @Override
clear(Repository repository)768     public void clear(Repository repository) {
769         String revPath = getRepositoryCachedRevPath(repository);
770         if (revPath != null) {
771             // remove the file cached last revision (done separately in case
772             // it gets ever moved outside the hierarchy)
773             File cachedRevFile = new File(revPath);
774             try {
775                 Files.delete(cachedRevFile.toPath());
776             } catch (IOException e) {
777                 LOGGER.log(Level.WARNING, String.format("failed to delete '%s'", cachedRevFile), e);
778             }
779         }
780 
781         String histDir = getRepositoryHistDataDirname(repository);
782         if (histDir != null) {
783             // Remove all files which constitute the history cache.
784             try {
785                 IOUtils.removeRecursive(Paths.get(histDir));
786             } catch (NoSuchFileException ex) {
787                 LOGGER.log(Level.WARNING, String.format("directory %s does not exist", histDir));
788             } catch (IOException ex) {
789                 LOGGER.log(Level.SEVERE, "tried removeRecursive()", ex);
790             }
791         }
792     }
793 
794     @Override
clearFile(String path)795     public void clearFile(String path) {
796         File historyFile;
797         try {
798             historyFile = getCachedFile(new File(env.getSourceRootPath() + path));
799         } catch (ForbiddenSymlinkException ex) {
800             LOGGER.log(Level.FINER, ex.getMessage());
801             return;
802         } catch (HistoryException ex) {
803             LOGGER.log(Level.WARNING, "cannot get history file for file " + path, ex);
804             return;
805         }
806         File parent = historyFile.getParentFile();
807 
808         if (!historyFile.delete() && historyFile.exists()) {
809             LOGGER.log(Level.WARNING,
810                 "Failed to remove obsolete history cache-file: {0}",
811                 historyFile.getAbsolutePath());
812         }
813 
814         if (parent.delete()) {
815             LOGGER.log(Level.FINE, "Removed empty history cache dir:{0}",
816                 parent.getAbsolutePath());
817         }
818     }
819 
820     @Override
getInfo()821     public String getInfo() {
822         return getClass().getSimpleName();
823     }
824 }
825