1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * See LICENSE.txt included in this distribution for the specific 9 * language governing permissions and limitations under the License. 10 * 11 * When distributing Covered Code, include this CDDL HEADER in each 12 * file and include the License file at LICENSE.txt. 13 * If applicable, add the following below this CDDL HEADER, with the 14 * fields enclosed by brackets "[]" replaced with your own identifying 15 * information: Portions Copyright [yyyy] [name of copyright owner] 16 * 17 * CDDL HEADER END 18 */ 19 20 /* 21 * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. 22 * Portions Copyright (c) 2020, Chris Fraire <cfraire@me.com>. 23 */ 24 package org.opengrok.web.api.v1.controller; 25 26 import static org.opengrok.indexer.history.RepositoryFactory.getRepository; 27 28 import java.io.File; 29 import java.io.IOException; 30 import java.nio.file.Paths; 31 import java.util.ArrayList; 32 import java.util.Collections; 33 import java.util.List; 34 import java.util.Map; 35 import java.util.Set; 36 import java.util.TreeSet; 37 import java.util.concurrent.CompletableFuture; 38 import java.util.logging.Level; 39 import java.util.logging.Logger; 40 import java.util.stream.Collectors; 41 42 import jakarta.inject.Inject; 43 import jakarta.servlet.http.HttpServletRequest; 44 import jakarta.ws.rs.Consumes; 45 import jakarta.ws.rs.DELETE; 46 import jakarta.ws.rs.GET; 47 import jakarta.ws.rs.NotFoundException; 48 import jakarta.ws.rs.POST; 49 import jakarta.ws.rs.PUT; 50 import jakarta.ws.rs.Path; 51 import jakarta.ws.rs.PathParam; 52 import jakarta.ws.rs.Produces; 53 import jakarta.ws.rs.WebApplicationException; 54 import jakarta.ws.rs.core.Context; 55 import jakarta.ws.rs.core.MediaType; 56 import jakarta.ws.rs.core.Response; 57 import org.opengrok.indexer.configuration.CommandTimeoutType; 58 import org.opengrok.indexer.configuration.Group; 59 import org.opengrok.indexer.configuration.Project; 60 import org.opengrok.indexer.configuration.RuntimeEnvironment; 61 import org.opengrok.indexer.history.HistoryGuru; 62 import org.opengrok.indexer.history.Repository; 63 import org.opengrok.indexer.history.RepositoryInfo; 64 import org.opengrok.indexer.index.IndexDatabase; 65 import org.opengrok.indexer.logger.LoggerFactory; 66 import org.opengrok.indexer.util.ClassUtil; 67 import org.opengrok.indexer.util.ForbiddenSymlinkException; 68 import org.opengrok.indexer.util.IOUtils; 69 import org.opengrok.web.api.ApiTask; 70 import org.opengrok.indexer.web.Laundromat; 71 import org.opengrok.web.api.ApiTaskManager; 72 import org.opengrok.web.api.v1.suggester.provider.service.SuggesterService; 73 74 @Path(ProjectsController.PROJECTS_PATH) 75 public class ProjectsController { 76 77 private static final Logger LOGGER = LoggerFactory.getLogger(ProjectsController.class); 78 79 public static final String PROJECTS_PATH = "/projects"; 80 81 private final RuntimeEnvironment env = RuntimeEnvironment.getInstance(); 82 83 @Inject 84 private SuggesterService suggester; 85 86 @POST 87 @Consumes(MediaType.TEXT_PLAIN) addProject(@ontext HttpServletRequest request, String projectNameParam)88 public Response addProject(@Context HttpServletRequest request, String projectNameParam) { 89 // Avoid classification as a taint bug. 90 final String projectName = Laundromat.launderInput(projectNameParam); 91 92 LOGGER.log(Level.INFO, "adding project {0}", projectName); 93 94 return ApiTaskManager.getInstance().submitApiTask(PROJECTS_PATH, 95 new ApiTask(request.getRequestURI(), 96 () -> { 97 addProjectWorkHorse(projectName); 98 return null; 99 }, 100 Response.Status.CREATED)); 101 } 102 addProjectWorkHorse(String projectName)103 private void addProjectWorkHorse(String projectName) { 104 File srcRoot = env.getSourceRootFile(); 105 File projDir = new File(srcRoot, projectName); 106 107 if (!env.getProjects().containsKey(projectName)) { 108 Project project = new Project(projectName, "/" + projectName); 109 110 if (env.isHistoryEnabled()) { 111 // Add repositories in this project. 112 List<RepositoryInfo> repos = getRepositoriesInDir(projDir); 113 114 env.addRepositories(repos); 115 env.getProjectRepositoriesMap().put(project, repos); 116 } 117 118 // Finally, introduce the project to the configuration. 119 // Note that the project is inactive in the UI until it is indexed. 120 // See isIndexed() 121 env.getProjects().put(projectName, project); 122 env.populateGroups(env.getGroups(), new TreeSet<>(env.getProjectList())); 123 } else { 124 Project project = env.getProjects().get(projectName); 125 Map<Project, List<RepositoryInfo>> map = env.getProjectRepositoriesMap(); 126 127 if (env.isHistoryEnabled()) { 128 // Refresh the list of repositories of this project. 129 // This is the goal of this action: if an existing project 130 // is re-added, this means its list of repositories has changed. 131 List<RepositoryInfo> repos = getRepositoriesInDir(projDir); 132 List<RepositoryInfo> allrepos = env.getRepositories(); 133 synchronized (allrepos) { 134 // newly added repository 135 for (RepositoryInfo repo : repos) { 136 if (!allrepos.contains(repo)) { 137 allrepos.add(repo); 138 } 139 } 140 // deleted repository 141 if (map.containsKey(project)) { 142 for (RepositoryInfo repo : map.get(project)) { 143 if (!repos.contains(repo)) { 144 allrepos.remove(repo); 145 } 146 } 147 } 148 } 149 150 map.put(project, repos); 151 } 152 } 153 } 154 getRepositoriesInDir(final File projDir)155 private List<RepositoryInfo> getRepositoriesInDir(final File projDir) { 156 157 HistoryGuru histGuru = HistoryGuru.getInstance(); 158 159 // There is no need to perform the work of invalidateRepositories(), 160 // since addRepositories() calls getRepository() for each of 161 // the repos. 162 return new ArrayList<>(histGuru.addRepositories(new File[]{projDir})); 163 } 164 disableProject(String projectName)165 private Project disableProject(String projectName) { 166 Project project = env.getProjects().get(projectName); 167 if (project == null) { 168 throw new IllegalStateException("cannot get project \"" + projectName + "\""); 169 } 170 171 // Remove the project from searches so no one can trip over incomplete index data. 172 project.setIndexed(false); 173 174 return project; 175 } 176 177 @DELETE 178 @Path("/{project}") deleteProject(@ontext HttpServletRequest request, @PathParam("project") String projectNameParam)179 public Response deleteProject(@Context HttpServletRequest request, @PathParam("project") String projectNameParam) { 180 // Avoid classification as a taint bug. 181 final String projectName = Laundromat.launderInput(projectNameParam); 182 183 Project project = disableProject(projectName); 184 LOGGER.log(Level.INFO, "deleting configuration for project {0}", projectName); 185 186 return ApiTaskManager.getInstance().submitApiTask(PROJECTS_PATH, 187 new ApiTask(request.getRequestURI(), 188 () -> { 189 deleteProjectWorkHorse(projectName, project); 190 return null; 191 }, 192 Response.Status.NO_CONTENT)); 193 } 194 195 private void deleteProjectWorkHorse(String projectName, Project project) { 196 // Delete index data associated with the project. 197 deleteProjectDataWorkHorse(projectName, true); 198 199 // Remove the project from its groups. 200 for (Group group : project.getGroups()) { 201 group.getRepositories().remove(project); 202 group.getProjects().remove(project); 203 } 204 205 if (env.isHistoryEnabled()) { 206 // Now remove the repositories associated with this project. 207 List<RepositoryInfo> repos = env.getProjectRepositoriesMap().get(project); 208 if (repos != null) { 209 env.getRepositories().removeAll(repos); 210 } 211 env.getProjectRepositoriesMap().remove(project); 212 } 213 214 env.getProjects().remove(projectName, project); 215 216 // Prevent the project to be included in new searches. 217 env.refreshSearcherManagerMap(); 218 } 219 220 @DELETE 221 @Path("/{project}/data") 222 public Response deleteProjectData(@Context HttpServletRequest request, 223 @PathParam("project") String projectNameParam) { 224 // Avoid classification as a taint bug. 225 final String projectName = Laundromat.launderInput(projectNameParam); 226 227 disableProject(projectName); 228 229 return ApiTaskManager.getInstance().submitApiTask(PROJECTS_PATH, 230 new ApiTask(request.getRequestURI(), 231 () -> { 232 deleteProjectDataWorkHorse(projectName, false); 233 return null; 234 }, 235 Response.Status.NO_CONTENT)); 236 } 237 238 private void deleteProjectDataWorkHorse(String projectName, boolean clearHistoryGuru) { 239 LOGGER.log(Level.INFO, "deleting data for project {0}", projectName); 240 241 // Delete index and xrefs. 242 for (String dirName: new String[]{IndexDatabase.INDEX_DIR, IndexDatabase.XREF_DIR}) { 243 java.nio.file.Path path = Paths.get(env.getDataRootPath(), dirName, projectName); 244 try { 245 IOUtils.removeRecursive(path); 246 } catch (IOException e) { 247 LOGGER.log(Level.WARNING, "Could not delete {0}", path); 248 } 249 } 250 251 deleteHistoryCacheWorkHorse(projectName, clearHistoryGuru); 252 253 // Delete suggester data. 254 suggester.delete(projectName); 255 } 256 257 @DELETE 258 @Path("/{project}/historycache") 259 public Response deleteHistoryCache(@Context HttpServletRequest request, 260 @PathParam("project") String projectNameParam) { 261 262 if (!env.isHistoryEnabled()) { 263 return Response.status(Response.Status.NO_CONTENT).build(); 264 } 265 266 // Avoid classification as a taint bug. 267 final String projectName = Laundromat.launderInput(projectNameParam); 268 269 return ApiTaskManager.getInstance().submitApiTask(PROJECTS_PATH, 270 new ApiTask(request.getRequestURI(), 271 () -> { 272 deleteHistoryCacheWorkHorse(projectName, false); 273 return null; 274 })); 275 } 276 277 private void deleteHistoryCacheWorkHorse(String projectName, boolean clearHistoryGuru) { 278 Project project = disableProject(projectName); 279 280 LOGGER.log(Level.INFO, "deleting history cache for project {0}", projectName); 281 282 List<RepositoryInfo> repos = env.getProjectRepositoriesMap().get(project); 283 if (repos == null || repos.isEmpty()) { 284 LOGGER.log(Level.INFO, "history cache for project {0} is not present", projectName); 285 return; 286 } 287 288 // Delete history cache data. 289 HistoryGuru guru = HistoryGuru.getInstance(); 290 guru.removeCache(repos.stream(). 291 map(x -> { 292 try { 293 return env.getPathRelativeToSourceRoot(new File((x).getDirectoryName())); 294 } catch (ForbiddenSymlinkException e) { 295 LOGGER.log(Level.FINER, e.getMessage()); 296 return ""; 297 } catch (IOException e) { 298 LOGGER.log(Level.WARNING, "cannot remove files for repository {0}", x.getDirectoryName()); 299 // Empty output should not cause any harm 300 // since {@code getReposFromString()} inside 301 // {@code removeCache()} will return nothing. 302 return ""; 303 } 304 }).filter(x -> !x.isEmpty()).collect(Collectors.toSet()), clearHistoryGuru); 305 } 306 307 @PUT 308 @Path("/{project}/indexed") 309 @Consumes(MediaType.TEXT_PLAIN) 310 public Response markIndexed(@Context HttpServletRequest request, @PathParam("project") String projectNameParam) { 311 312 // Avoid classification as a taint bug. 313 final String projectName = Laundromat.launderInput(projectNameParam); 314 315 Project project = env.getProjects().get(projectName); 316 if (project == null) { 317 LOGGER.log(Level.WARNING, "cannot find project {0} to mark as indexed", projectName); 318 throw new NotFoundException(String.format("project '%s' does not exist", projectName)); 319 } 320 321 project.setIndexed(true); 322 323 return ApiTaskManager.getInstance().submitApiTask(PROJECTS_PATH, 324 new ApiTask(request.getRequestURI(), 325 () -> { 326 // Refresh current version of the project's repositories. 327 List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project); 328 if (riList != null) { 329 for (RepositoryInfo ri : riList) { 330 Repository repo = getRepository(ri, CommandTimeoutType.RESTFUL); 331 332 if (repo != null && repo.getCurrentVersion() != null && 333 repo.getCurrentVersion().length() > 0) { 334 // getRepository() always creates fresh instance 335 // of the Repository object so there is no need 336 // to call setCurrentVersion() on it. 337 ri.setCurrentVersion(repo.determineCurrentVersion()); 338 } 339 } 340 } 341 342 CompletableFuture.runAsync(() -> suggester.rebuild(projectName)); 343 344 // In case this project has just been incrementally indexed, 345 // its IndexSearcher needs a poke. 346 env.maybeRefreshIndexSearchers(Collections.singleton(projectName)); 347 348 env.refreshDateForLastIndexRun(); 349 return null; 350 })); 351 } 352 353 @PUT 354 @Path("/{project}/property/{field}") 355 public void set( 356 @PathParam("project") String projectName, 357 @PathParam("field") String field, 358 final String value 359 ) throws IOException { 360 // Avoid classification as a taint bug. 361 projectName = Laundromat.launderInput(projectName); 362 field = Laundromat.launderInput(field); 363 364 Project project = env.getProjects().get(projectName); 365 if (project != null) { 366 // Set the property. 367 ClassUtil.setFieldValue(project, field, value); 368 369 // Refresh field values for project's repositories for this project as well. 370 List<RepositoryInfo> riList = env.getProjectRepositoriesMap().get(project); 371 if (riList != null) { 372 for (RepositoryInfo ri : riList) { 373 // Set the property if there is one. 374 if (ClassUtil.hasField(ri, field)) { 375 ClassUtil.setFieldValue(ri, field, value); 376 } 377 } 378 } 379 } else { 380 LOGGER.log(Level.WARNING, "cannot find project {0} to set a property", projectName); 381 } 382 } 383 384 @GET 385 @Path("/{project}/property/{field}") 386 @Produces(MediaType.APPLICATION_JSON) 387 public Object get(@PathParam("project") String projectName, @PathParam("field") String field) 388 throws IOException { 389 // Avoid classification as a taint bug. 390 projectName = Laundromat.launderInput(projectName); 391 field = Laundromat.launderInput(field); 392 393 Project project = env.getProjects().get(projectName); 394 if (project == null) { 395 throw new WebApplicationException( 396 "cannot find project " + projectName + " to get a property", Response.Status.BAD_REQUEST); 397 } 398 return ClassUtil.getFieldValue(project, field); 399 } 400 401 @GET 402 @Produces(MediaType.APPLICATION_JSON) 403 public List<String> listProjects() { 404 return env.getProjectNames(); 405 } 406 407 @GET 408 @Path("indexed") 409 @Produces(MediaType.APPLICATION_JSON) 410 public List<String> listIndexed() { 411 return env.getProjectList().stream() 412 .filter(Project::isIndexed) 413 .map(Project::getName) 414 .collect(Collectors.toList()); 415 } 416 417 @GET 418 @Path("/{project}/repositories") 419 @Produces(MediaType.APPLICATION_JSON) 420 public List<String> getRepositories(@PathParam("project") String projectName) { 421 // Avoid classification as a taint bug. 422 projectName = Laundromat.launderInput(projectName); 423 424 Project project = env.getProjects().get(projectName); 425 if (project != null) { 426 List<RepositoryInfo> infos = env.getProjectRepositoriesMap().get(project); 427 if (infos != null) { 428 return infos.stream() 429 .map(RepositoryInfo::getDirectoryNameRelative) 430 .collect(Collectors.toList()); 431 } 432 } 433 434 return Collections.emptyList(); 435 } 436 437 @GET 438 @Path("/{project}/repositories/type") 439 @Produces(MediaType.APPLICATION_JSON) 440 public Set<String> getRepositoriesType(@PathParam("project") String projectName) { 441 // Avoid classification as a taint bug. 442 projectName = Laundromat.launderInput(projectName); 443 444 Project project = env.getProjects().get(projectName); 445 if (project != null) { 446 List<RepositoryInfo> infos = env.getProjectRepositoriesMap().get(project); 447 if (infos != null) { 448 return infos.stream() 449 .map(RepositoryInfo::getType) 450 .collect(Collectors.toSet()); 451 } 452 } 453 return Collections.emptySet(); 454 } 455 456 @GET 457 @Path("/{project}/files") 458 @Produces(MediaType.APPLICATION_JSON) 459 public Set<String> getProjectIndexFiles(@PathParam("project") String projectName) throws IOException { 460 // Avoid classification as a taint bug. 461 projectName = Laundromat.launderInput(projectName); 462 463 return IndexDatabase.getAllFiles(Collections.singletonList("/" + projectName)); 464 } 465 } 466