1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * See LICENSE.txt included in this distribution for the specific 9 * language governing permissions and limitations under the License. 10 * 11 * When distributing Covered Code, include this CDDL HEADER in each 12 * file and include the License file at LICENSE.txt. 13 * If applicable, add the following below this CDDL HEADER, with the 14 * fields enclosed by brackets "[]" replaced with your own identifying 15 * information: Portions Copyright [yyyy] [name of copyright owner] 16 * 17 * CDDL HEADER END 18 */ 19 20 /* 21 * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. 22 * Portions Copyright (c) 2019, 2020, Chris Fraire <cfraire@me.com>. 23 */ 24 package org.opengrok.web.api.v1.controller; 25 26 import jakarta.ws.rs.client.Entity; 27 import jakarta.ws.rs.core.GenericType; 28 import jakarta.ws.rs.core.Response; 29 import org.glassfish.jersey.internal.inject.AbstractBinder; 30 import org.glassfish.jersey.server.ResourceConfig; 31 import org.glassfish.jersey.servlet.ServletContainer; 32 import org.glassfish.jersey.test.DeploymentContext; 33 import org.glassfish.jersey.test.ServletDeploymentContext; 34 import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; 35 import org.glassfish.jersey.test.spi.TestContainerException; 36 import org.glassfish.jersey.test.spi.TestContainerFactory; 37 import org.junit.jupiter.api.AfterEach; 38 import org.junit.jupiter.api.BeforeAll; 39 import org.junit.jupiter.api.BeforeEach; 40 import org.junit.jupiter.api.Test; 41 import org.junit.jupiter.api.extension.ExtendWith; 42 import org.mockito.Mock; 43 import org.mockito.junit.jupiter.MockitoExtension; 44 import org.opengrok.indexer.condition.EnabledForRepository; 45 import org.opengrok.indexer.configuration.CommandTimeoutType; 46 import org.opengrok.indexer.configuration.Group; 47 import org.opengrok.indexer.configuration.Project; 48 import org.opengrok.indexer.configuration.RuntimeEnvironment; 49 import org.opengrok.indexer.history.HistoryGuru; 50 import org.opengrok.indexer.history.MercurialRepositoryTest; 51 import org.opengrok.indexer.history.RepositoryFactory; 52 import org.opengrok.indexer.history.RepositoryInfo; 53 import org.opengrok.indexer.index.IndexDatabase; 54 import org.opengrok.indexer.index.Indexer; 55 import org.opengrok.indexer.index.IndexerException; 56 import org.opengrok.indexer.util.TestRepository; 57 import org.opengrok.web.api.ApiTaskManager; 58 import org.opengrok.web.api.v1.suggester.provider.service.SuggesterService; 59 60 import java.io.File; 61 import java.io.IOException; 62 import java.nio.file.Files; 63 import java.nio.file.Path; 64 import java.nio.file.Paths; 65 import java.nio.file.StandardCopyOption; 66 import java.util.ArrayList; 67 import java.util.Arrays; 68 import java.util.Collections; 69 import java.util.List; 70 import java.util.Set; 71 import java.util.concurrent.ConcurrentHashMap; 72 import java.util.stream.Collectors; 73 74 import static org.junit.jupiter.api.Assertions.assertEquals; 75 import static org.junit.jupiter.api.Assertions.assertFalse; 76 import static org.junit.jupiter.api.Assertions.assertNotNull; 77 import static org.junit.jupiter.api.Assertions.assertTrue; 78 import static org.opengrok.indexer.condition.RepositoryInstalled.Type.MERCURIAL; 79 import static org.opengrok.indexer.condition.RepositoryInstalled.Type.SUBVERSION; 80 import static org.opengrok.indexer.util.IOUtils.removeRecursive; 81 import static org.opengrok.web.api.v1.controller.ApiUtils.waitForTask; 82 83 @ExtendWith(MockitoExtension.class) 84 @EnabledForRepository({MERCURIAL, SUBVERSION}) 85 class ProjectsControllerTest extends OGKJerseyTest { 86 87 private final RuntimeEnvironment env = RuntimeEnvironment.getInstance(); 88 89 private TestRepository repository; 90 91 @Mock 92 private SuggesterService suggesterService; 93 94 @BeforeAll setup()95 static void setup() { 96 ApiTaskManager.getInstance().addPool("projects", 1); 97 } 98 99 @Override getTestContainerFactory()100 protected TestContainerFactory getTestContainerFactory() throws TestContainerException { 101 return new GrizzlyWebTestContainerFactory(); 102 } 103 104 @Override configureDeployment()105 protected DeploymentContext configureDeployment() { 106 return ServletDeploymentContext.forServlet(new ServletContainer(new ResourceConfig(ProjectsController.class) 107 .register(new AbstractBinder() { 108 @Override 109 protected void configure() { 110 bind(suggesterService).to(SuggesterService.class); 111 } 112 }))).build(); 113 } 114 115 @BeforeEach 116 @Override 117 public void setUp() throws Exception { 118 super.setUp(); 119 repository = new TestRepository(); 120 repository.create(HistoryGuru.class.getResource("/repositories")); 121 122 env.setSourceRoot(repository.getSourceRoot()); 123 env.setDataRoot(repository.getDataRoot()); 124 env.setProjectsEnabled(true); 125 env.setHistoryEnabled(true); 126 env.setHandleHistoryOfRenamedFiles(true); 127 RepositoryFactory.initializeIgnoredNames(env); 128 } 129 130 @AfterEach 131 @Override 132 public void tearDown() throws Exception { 133 super.tearDown(); 134 // This should match Configuration constructor. 135 env.setProjects(new ConcurrentHashMap<>()); 136 env.setRepositories(new ArrayList<>()); 137 env.getProjectRepositoriesMap().clear(); 138 139 repository.destroy(); 140 } 141 142 @Test 143 void testAddInherit() { 144 assertTrue(env.getRepositories().isEmpty()); 145 assertTrue(env.getProjects().isEmpty()); 146 assertTrue(env.isHandleHistoryOfRenamedFiles()); 147 148 addProject("git"); 149 150 assertTrue(env.getProjects().containsKey("git")); 151 assertEquals(1, env.getProjects().size()); 152 153 Project proj = env.getProjects().get("git"); 154 assertNotNull(proj); 155 assertTrue(proj.isHandleRenamedFiles()); 156 } 157 158 private Response addProject(final String project) { 159 Response response = target("projects") 160 .request() 161 .post(Entity.text(project)); 162 return waitForTask(response); 163 } 164 165 /** 166 * Verify that added project correctly inherits a property 167 * from configuration. Ideally, this should test all properties of Project. 168 */ 169 @Test 170 void testAdd() throws Exception { 171 assertTrue(env.getRepositories().isEmpty()); 172 assertTrue(env.getProjects().isEmpty()); 173 174 // Add a group matching the project to be added. 175 String groupName = "mercurialgroup"; 176 Group group = new Group(groupName, "mercurial.*"); 177 env.getGroups().add(group); 178 assertTrue(env.hasGroups()); 179 assertEquals(1, env.getGroups().stream(). 180 filter(g -> g.getName().equals(groupName)). 181 collect(Collectors.toSet()).size()); 182 assertEquals(0, group.getRepositories().size()); 183 assertEquals(0, group.getProjects().size()); 184 185 // Add a sub-repository. 186 String repoPath = repository.getSourceRoot() + File.separator + "mercurial"; 187 File mercurialRoot = new File(repoPath); 188 File subDir = new File(mercurialRoot, "usr"); 189 assertTrue(subDir.mkdir()); 190 String subRepoPath = repoPath + File.separator + "usr" + File.separator + "closed"; 191 File mercurialSubRoot = new File(subRepoPath); 192 MercurialRepositoryTest.runHgCommand(mercurialRoot, 193 "clone", mercurialRoot.getAbsolutePath(), subRepoPath); 194 195 // Add the project. 196 env.setScanningDepth(3); 197 198 addProject("mercurial"); 199 200 // Check that the project was added properly. 201 assertTrue(env.getProjects().containsKey("mercurial")); 202 assertEquals(1, env.getProjects().size()); 203 assertEquals(2, env.getRepositories().size()); 204 assertEquals(1, group.getRepositories().size()); 205 assertEquals(0, group.getProjects().size()); 206 assertEquals(1, group.getRepositories().stream(). 207 filter(p -> p.getName().equals("mercurial")). 208 collect(Collectors.toSet()).size()); 209 210 // Check that HistoryGuru now includes the project in its list. 211 Set<String> directoryNames = HistoryGuru.getInstance(). 212 getRepositories().stream().map(RepositoryInfo::getDirectoryName). 213 collect(Collectors.toSet()); 214 assertTrue(directoryNames.contains(repoPath) || directoryNames.contains( 215 mercurialRoot.getCanonicalPath()), "though it should contain the top root,"); 216 assertTrue(directoryNames.contains(subRepoPath) || directoryNames.contains( 217 mercurialSubRoot.getCanonicalPath()), "though it should contain the sub-root,"); 218 219 // Add more projects and check that they have been added incrementally. 220 // At the same time, it checks that multiple projects can be added 221 // with single message. 222 223 addProject("git"); 224 225 assertEquals(2, env.getProjects().size()); 226 assertEquals(3, env.getRepositories().size()); 227 assertTrue(env.getProjects().containsKey("git")); 228 229 assertFalse(HistoryGuru.getInstance().getRepositories().stream(). 230 map(RepositoryInfo::getDirectoryName).collect(Collectors.toSet()). 231 contains("git")); 232 } 233 234 /** 235 * Test that if the add is applied on already existing project, 236 * the repository list is refreshed. 237 */ 238 @Test 239 void testRepositoryRefresh() throws Exception { 240 addProject("mercurial"); 241 242 File mercurialRoot = new File(repository.getSourceRoot() + File.separator + "mercurial"); 243 MercurialRepositoryTest.runHgCommand(mercurialRoot, 244 "clone", mercurialRoot.getAbsolutePath(), 245 mercurialRoot.getAbsolutePath() + File.separator + "closed"); 246 247 addProject("mercurial"); 248 249 assertEquals(2, env.getRepositories().size()); 250 assertEquals(2, env.getProjectRepositoriesMap().get(Project.getProject(mercurialRoot)).size()); 251 252 // Delete the newly added repository to verify it will be removed from 253 // configuration after the message is reapplied. This is necessary anyway 254 // for proper per-test cleanup. 255 removeRecursive(new File(mercurialRoot.getAbsolutePath() + File.separator + "closed").toPath()); 256 257 addProject("mercurial"); 258 259 assertEquals(1, env.getRepositories().size()); 260 assertEquals(1, env.getProjectRepositoriesMap().get(Project.getProject(mercurialRoot)).size()); 261 } 262 263 /** 264 * This test needs to perform indexing so that it can be verified that 265 * delete handling does remove the index data. 266 */ 267 @Test 268 void testDelete() throws Exception { 269 String[] projectsToDelete = {"git"}; 270 271 // Add a group matching the project to be added. 272 String groupName = "gitgroup"; 273 Group group = new Group(groupName, "git.*"); 274 env.getGroups().add(group); 275 assertTrue(env.hasGroups()); 276 assertEquals(1, env.getGroups().stream(). 277 filter(g -> g.getName().equals(groupName)). 278 collect(Collectors.toSet()).size()); 279 assertEquals(0, group.getRepositories().size()); 280 assertEquals(0, group.getProjects().size()); 281 282 assertEquals(0, env.getProjects().size()); 283 assertEquals(0, env.getRepositories().size()); 284 assertEquals(0, env.getProjectRepositoriesMap().size()); 285 286 addProject("mercurial"); 287 addProject("git"); 288 289 assertEquals(2, env.getProjects().size()); 290 assertEquals(2, env.getRepositories().size()); 291 assertEquals(2, env.getProjectRepositoriesMap().size()); 292 293 // Check the group was populated properly. 294 assertEquals(1, group.getRepositories().size()); 295 assertEquals(0, group.getProjects().size()); 296 assertEquals(1, group.getRepositories().stream(). 297 filter(p -> p.getName().equals("git")). 298 collect(Collectors.toSet()).size()); 299 300 // Run the indexer so that data directory is populated. 301 ArrayList<String> subFiles = new ArrayList<>(); 302 subFiles.add("/git"); 303 subFiles.add("/mercurial"); 304 ArrayList<String> repos = new ArrayList<>(); 305 repos.add("/git"); 306 repos.add("/mercurial"); 307 // This is necessary so that repositories in HistoryGuru get populated. 308 // For per project reindex this is called from setConfiguration() because 309 // of the -R option is present. 310 HistoryGuru.getInstance().invalidateRepositories( 311 env.getRepositories(), null, CommandTimeoutType.INDEXER); 312 env.setHistoryEnabled(true); 313 Indexer.getInstance().prepareIndexer( 314 env, 315 false, // don't search for repositories 316 false, // don't scan and add projects 317 false, // don't create dictionary 318 subFiles, // subFiles - needed when refreshing history partially 319 repos); // repositories - needed when refreshing history partially 320 Indexer.getInstance().doIndexerExecution(true, null, null); 321 322 for (String proj : projectsToDelete) { 323 deleteProject(proj); 324 } 325 326 assertEquals(1, env.getProjects().size()); 327 assertEquals(1, env.getRepositories().size()); 328 assertEquals(1, env.getProjectRepositoriesMap().size()); 329 330 // Test data removal. 331 for (String projectName : projectsToDelete) { 332 for (String dirName : new String[] {"historycache", 333 IndexDatabase.XREF_DIR, IndexDatabase.INDEX_DIR}) { 334 File dir = new File(env.getDataRootFile(), 335 dirName + File.separator + projectName); 336 assertFalse(dir.exists()); 337 } 338 } 339 340 // Check that HistoryGuru no longer maintains the removed projects. 341 for (String p : projectsToDelete) { 342 assertFalse(HistoryGuru.getInstance().getRepositories().stream(). 343 map(RepositoryInfo::getDirectoryName).collect(Collectors.toSet()). 344 contains(repository.getSourceRoot() + File.separator + p)); 345 } 346 347 // Check the group no longer contains the removed project. 348 assertEquals(0, group.getRepositories().size()); 349 assertEquals(0, group.getProjects().size()); 350 } 351 352 private Response deleteProject(final String project) { 353 Response response = target("projects") 354 .path(project) 355 .request() 356 .delete(); 357 return waitForTask(response); 358 } 359 360 @Test 361 void testIndexed() throws IOException { 362 String projectName = "mercurial"; 363 364 // When a project is added, it should be marked as not indexed. 365 addProject(projectName); 366 367 assertFalse(env.getProjects().get(projectName).isIndexed()); 368 369 // Get repository info for the project. 370 Project project = env.getProjects().get(projectName); 371 assertNotNull(project); 372 List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project); 373 assertNotNull(riList); 374 assertEquals(1, riList.size(), "there should be just 1 repository"); 375 RepositoryInfo ri = riList.get(0); 376 assertNotNull(ri); 377 assertTrue(ri.getCurrentVersion().contains("8b340409b3a8")); 378 379 // Add some changes to the repository. 380 File mercurialRoot = new File(repository.getSourceRoot() + File.separator + "mercurial"); 381 382 // copy file from jar to a temp file 383 Path temp = Files.createTempFile("opengrok", "temp"); 384 Files.copy(HistoryGuru.getInstance().getClass().getResourceAsStream("/history/hg-export-subdir.txt"), 385 temp, StandardCopyOption.REPLACE_EXISTING); 386 387 // prevent 'uncommitted changes' error 388 MercurialRepositoryTest.runHgCommand(mercurialRoot, "revert", "--all"); 389 390 MercurialRepositoryTest.runHgCommand(mercurialRoot, "import", temp.toString()); 391 392 assertTrue(temp.toFile().delete()); 393 394 // Test that the project's indexed flag becomes true only after 395 // the message is applied. 396 397 398 assertEquals(markIndexed(projectName).getStatusInfo().getFamily(), Response.Status.Family.SUCCESSFUL); 399 assertTrue(env.getProjects().get(projectName).isIndexed(), "indexed flag should be set to true"); 400 401 // Test that the "indexed" message triggers refresh of current version 402 // info in related repositories. 403 riList = env.getProjectRepositoriesMap().get(project); 404 assertNotNull(riList); 405 ri = riList.get(0); 406 assertNotNull(ri); 407 assertTrue(ri.getCurrentVersion().contains("c78fa757c524"), "current version should be refreshed"); 408 } 409 410 private Response markIndexed(final String project) { 411 Response response = target("projects") 412 .path(project) 413 .path("indexed") 414 .request() 415 .put(Entity.text("")); 416 return waitForTask(response); 417 } 418 419 @Test 420 void testList() { 421 addProject("mercurial"); 422 assertEquals(markIndexed("mercurial").getStatusInfo().getFamily(), Response.Status.Family.SUCCESSFUL); 423 424 // Add another project. 425 addProject("git"); 426 427 GenericType<List<String>> type = new GenericType<>() { 428 }; 429 430 List<String> projects = target("projects") 431 .request() 432 .get(type); 433 434 assertTrue(projects.contains("mercurial")); 435 assertTrue(projects.contains("git")); 436 437 List<String> indexed = target("projects") 438 .path("indexed") 439 .request() 440 .get(type); 441 442 assertTrue(indexed.contains("mercurial")); 443 assertFalse(indexed.contains("git")); 444 } 445 446 @Test 447 void testGetReposForNonExistentProject() { 448 GenericType<List<String>> type = new GenericType<>() { 449 }; 450 451 // Try to get repos for non-existent project first. 452 List<String> repos = target("projects") 453 .path("totally-nonexistent-project") 454 .path("repositories") 455 .request() 456 .get(type); 457 458 assertTrue(repos.isEmpty()); 459 } 460 461 @Test 462 void testGetRepos() throws Exception { 463 GenericType<List<String>> type = new GenericType<>() { 464 }; 465 466 // Create subrepository. 467 File mercurialRoot = new File(repository.getSourceRoot() + File.separator + "mercurial"); 468 MercurialRepositoryTest.runHgCommand(mercurialRoot, 469 "clone", mercurialRoot.getAbsolutePath(), 470 mercurialRoot.getAbsolutePath() + File.separator + "closed"); 471 472 addProject("mercurial"); 473 474 // Get repositories of the project. 475 List<String> repos = target("projects") 476 .path("mercurial") 477 .path("repositories") 478 .request() 479 .get(type); 480 481 // Perform cleanup of the subrepository in order not to interfere 482 // with other tests. 483 removeRecursive(new File(mercurialRoot.getAbsolutePath() + 484 File.separator + "closed").toPath()); 485 486 // test 487 assertEquals( 488 new ArrayList<>(Arrays.asList(Paths.get("/mercurial").toString(), Paths.get("/mercurial/closed").toString())), 489 repos); 490 491 // Test the types. There should be only one type for project with 492 // multiple nested Mercurial repositories. 493 494 List<String> types = target("projects") 495 .path("mercurial") 496 .path("repositories/type") 497 .request() 498 .get(type); 499 500 assertEquals(Collections.singletonList("Mercurial"), types); 501 } 502 503 @Test 504 void testSetIndexed() { 505 String project = "git"; 506 addProject(project); 507 assertEquals(1, env.getProjectList().size()); 508 509 env.getProjects().get(project).setIndexed(false); 510 assertFalse(env.getProjects().get(project).isIndexed()); 511 Response response = target("projects") 512 .path(project) 513 .path("property/indexed") 514 .request() 515 .put(Entity.text(Boolean.TRUE.toString())); 516 assertEquals(response.getStatus(), Response.Status.NO_CONTENT.getStatusCode()); 517 assertTrue(env.getProjects().get(project).isIndexed()); 518 } 519 520 @Test 521 void testSetGet() { 522 assertTrue(env.isHandleHistoryOfRenamedFiles()); 523 String[] projects = new String[] {"mercurial", "git"}; 524 525 for (String proj : projects) { 526 addProject(proj); 527 } 528 529 assertEquals(2, env.getProjectList().size()); 530 for (String proj : projects) { 531 Project project = env.getProjects().get(proj); 532 assertNotNull(project); 533 assertTrue(project.isHandleRenamedFiles()); 534 List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project); 535 assertNotNull(riList); 536 for (RepositoryInfo ri : riList) { 537 ri.setHandleRenamedFiles(true); 538 assertTrue(ri.isHandleRenamedFiles()); 539 } 540 } 541 542 // Change their property via RESTful API call. 543 for (String proj : projects) { 544 setHandleRenamedFilesToFalse(proj); 545 } 546 547 // Verify the property was set on each project and its repositories. 548 for (String proj : projects) { 549 Project project = env.getProjects().get(proj); 550 assertNotNull(project); 551 assertFalse(project.isHandleRenamedFiles()); 552 List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project); 553 assertNotNull(riList); 554 for (RepositoryInfo ri : riList) { 555 assertFalse(ri.isHandleRenamedFiles()); 556 } 557 } 558 559 // Verify the property can be retrieved via message. 560 for (String proj : projects) { 561 boolean value = target("projects") 562 .path(proj) 563 .path("property/handleRenamedFiles") 564 .request() 565 .get(boolean.class); 566 assertFalse(value); 567 } 568 } 569 570 private void setHandleRenamedFilesToFalse(final String project) { 571 target("projects") 572 .path(project) 573 .path("property/handleRenamedFiles") 574 .request() 575 .put(Entity.text(Boolean.FALSE.toString())); 576 } 577 578 @Test 579 void testListFiles() throws IOException, IndexerException { 580 final String projectName = "mercurial"; 581 GenericType<List<String>> type = new GenericType<>() { 582 }; 583 584 Indexer.getInstance().prepareIndexer( 585 env, 586 false, // don't search for repositories 587 true, // add projects 588 false, // don't create dictionary 589 new ArrayList<>(), // subFiles - needed when refreshing history partially 590 new ArrayList<>()); // repositories - needed when refreshing history partially 591 Indexer.getInstance().doIndexerExecution(true, null, null); 592 593 List<String> filesFromRequest = target("projects") 594 .path(projectName) 595 .path("files") 596 .request() 597 .get(type); 598 filesFromRequest.sort(String::compareTo); 599 String[] files = {"Makefile", "bar.txt", "header.h", "main.c", "novel.txt"}; 600 for (int i = 0; i < files.length; i++) { 601 files[i] = "/" + projectName + "/" + files[i]; 602 } 603 List<String> expectedFiles = Arrays.asList(files); 604 expectedFiles.sort(String::compareTo); 605 606 assertEquals(expectedFiles, filesFromRequest); 607 } 608 } 609