1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * See LICENSE.txt included in this distribution for the specific 9 * language governing permissions and limitations under the License. 10 * 11 * When distributing Covered Code, include this CDDL HEADER in each 12 * file and include the License file at LICENSE.txt. 13 * If applicable, add the following below this CDDL HEADER, with the 14 * fields enclosed by brackets "[]" replaced with your own identifying 15 * information: Portions Copyright [yyyy] [name of copyright owner] 16 * 17 * CDDL HEADER END 18 */ 19 20 /* 21 * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. 22 * Portions Copyright (c) 2019, Chris Fraire <cfraire@me.com>. 23 */ 24 package org.opengrok.indexer.util; 25 26 import java.io.BufferedInputStream; 27 import java.io.ByteArrayInputStream; 28 import java.io.ByteArrayOutputStream; 29 import java.io.File; 30 import java.io.IOException; 31 import java.io.InputStream; 32 import java.io.InputStreamReader; 33 import java.io.Reader; 34 import java.lang.Thread.UncaughtExceptionHandler; 35 import java.util.Arrays; 36 import java.util.List; 37 import java.util.Map; 38 import java.util.Timer; 39 import java.util.TimerTask; 40 import java.util.logging.Level; 41 import java.util.logging.Logger; 42 import java.util.regex.Matcher; 43 import java.util.regex.Pattern; 44 45 import org.apache.commons.lang3.SystemUtils; 46 import org.opengrok.indexer.configuration.RuntimeEnvironment; 47 import org.opengrok.indexer.logger.LoggerFactory; 48 49 /** 50 * Wrapper to Java Process API. 51 * 52 * @author Emilio Monti - emilmont@gmail.com 53 */ 54 public class Executor { 55 56 private static final Logger LOGGER = LoggerFactory.getLogger(Executor.class); 57 58 private static final Pattern ARG_WIN_QUOTING = Pattern.compile("[^-:.+=%a-zA-Z0-9_/\\\\]"); 59 private static final Pattern ARG_UNIX_QUOTING = Pattern.compile("[^-:.+=%a-zA-Z0-9_/]"); 60 private static final Pattern ARG_GNU_STYLE_EQ = Pattern.compile("^--[-.a-zA-Z0-9_]+="); 61 62 private final List<String> cmdList; 63 private final File workingDirectory; 64 private byte[] stdout; 65 private byte[] stderr; 66 private int timeout; // in milliseconds, 0 means no timeout 67 68 /** 69 * Create a new instance of the Executor. 70 * @param cmd An array containing the command to execute 71 */ Executor(String[] cmd)72 public Executor(String[] cmd) { 73 this(Arrays.asList(cmd)); 74 } 75 76 /** 77 * Create a new instance of the Executor. 78 * @param cmdList A list containing the command to execute 79 */ Executor(List<String> cmdList)80 public Executor(List<String> cmdList) { 81 this(cmdList, null); 82 } 83 84 /** 85 * Create a new instance of the Executor with default command timeout value. 86 * The timeout value will be based on the running context (indexer or web application). 87 * @param cmdList A list containing the command to execute 88 * @param workingDirectory The directory the process should have as the 89 * working directory 90 */ Executor(List<String> cmdList, File workingDirectory)91 public Executor(List<String> cmdList, File workingDirectory) { 92 RuntimeEnvironment env = RuntimeEnvironment.getInstance(); 93 int timeoutSec = env.isIndexer() ? env.getIndexerCommandTimeout() : env.getInteractiveCommandTimeout(); 94 95 this.cmdList = cmdList; 96 this.workingDirectory = workingDirectory; 97 this.timeout = timeoutSec * 1000; 98 } 99 100 /** 101 * Create a new instance of the Executor with specific timeout value. 102 * @param cmdList A list containing the command to execute 103 * @param workingDirectory The directory the process should have as the 104 * working directory 105 * @param timeout If the command runs longer than the timeout (seconds), 106 * it will be terminated. If the value is 0, no timer 107 * will be set up. 108 */ Executor(List<String> cmdList, File workingDirectory, int timeout)109 public Executor(List<String> cmdList, File workingDirectory, int timeout) { 110 this.cmdList = cmdList; 111 this.workingDirectory = workingDirectory; 112 this.timeout = timeout * 1000; 113 } 114 115 /** 116 * Create a new instance of the Executor with or without timeout. 117 * @param cmdList A list containing the command to execute 118 * @param workingDirectory The directory the process should have as the 119 * working directory 120 * @param useTimeout terminate the process after default timeout or not 121 */ Executor(List<String> cmdList, File workingDirectory, boolean useTimeout)122 public Executor(List<String> cmdList, File workingDirectory, boolean useTimeout) { 123 this(cmdList, workingDirectory); 124 if (!useTimeout) { 125 this.timeout = 0; 126 } 127 } 128 129 /** 130 * Execute the command and collect the output. All exceptions will be 131 * logged. 132 * 133 * @return The exit code of the process 134 */ exec()135 public int exec() { 136 return exec(true); 137 } 138 139 /** 140 * Execute the command and collect the output. 141 * 142 * @param reportExceptions Should exceptions be added to the log or not 143 * @return The exit code of the process 144 */ exec(boolean reportExceptions)145 public int exec(boolean reportExceptions) { 146 SpoolHandler spoolOut = new SpoolHandler(); 147 int ret = exec(reportExceptions, spoolOut); 148 stdout = spoolOut.getBytes(); 149 return ret; 150 } 151 152 /** 153 * Execute the command and collect the output. 154 * 155 * @param reportExceptions Should exceptions be added to the log or not 156 * @param handler The handler to handle data from standard output 157 * @return The exit code of the process 158 */ exec(final boolean reportExceptions, StreamHandler handler)159 public int exec(final boolean reportExceptions, StreamHandler handler) { 160 int ret = -1; 161 ProcessBuilder processBuilder = new ProcessBuilder(cmdList); 162 final String cmd_str = escapeForShell(processBuilder.command(), false, SystemUtils.IS_OS_WINDOWS); 163 final String dir_str; 164 Timer timer = null; // timer for timing out the process 165 166 if (workingDirectory != null) { 167 processBuilder.directory(workingDirectory); 168 if (processBuilder.environment().containsKey("PWD")) { 169 processBuilder.environment().put("PWD", 170 workingDirectory.getAbsolutePath()); 171 } 172 } 173 174 File cwd = processBuilder.directory(); 175 if (cwd == null) { 176 dir_str = System.getProperty("user.dir"); 177 } else { 178 dir_str = cwd.toString(); 179 } 180 181 String envStr = ""; 182 if (LOGGER.isLoggable(Level.FINER)) { 183 Map<String, String> envMap = processBuilder.environment(); 184 envStr = " with environment: " + envMap.toString(); 185 } 186 LOGGER.log(Level.FINE, 187 "Executing command [{0}] in directory {1}{2}", 188 new Object[] {cmd_str, dir_str, envStr}); 189 190 Process process = null; 191 try { 192 Statistics stat = new Statistics(); 193 process = processBuilder.start(); 194 final Process proc = process; 195 196 final InputStream errorStream = process.getErrorStream(); 197 final SpoolHandler err = new SpoolHandler(); 198 Thread thread = new Thread(() -> { 199 try { 200 err.processStream(errorStream); 201 } catch (IOException ex) { 202 if (reportExceptions) { 203 LOGGER.log(Level.SEVERE, 204 "Error while executing command [{0}] in directory {1}", 205 new Object[] {cmd_str, dir_str}); 206 LOGGER.log(Level.SEVERE, "Error during process pipe listening", ex); 207 } 208 } 209 }); 210 thread.start(); 211 212 int timeout = this.timeout; 213 /* 214 * Setup timer so if the process get stuck we can terminate it and 215 * make progress instead of hanging the whole operation. 216 */ 217 if (timeout != 0) { 218 // invoking the constructor starts the background thread 219 timer = new Timer(); 220 timer.schedule(new TimerTask() { 221 @Override public void run() { 222 LOGGER.log(Level.WARNING, 223 String.format("Terminating process of command [%s] in directory %s " + 224 "due to timeout %d seconds", cmd_str, dir_str, timeout / 1000)); 225 proc.destroy(); 226 } 227 }, timeout); 228 } 229 230 handler.processStream(process.getInputStream()); 231 232 ret = process.waitFor(); 233 234 stat.report(LOGGER, Level.FINE, 235 String.format("Finished command [%s] in directory %s with exit code %d", cmd_str, dir_str, ret), 236 "executor.latency"); 237 LOGGER.log(Level.FINE, 238 "Finished command [{0}] in directory {1} with exit code {2}", 239 new Object[] {cmd_str, dir_str, ret}); 240 241 // Wait for the stderr read-out thread to finish the processing and 242 // only after that read the data. 243 thread.join(); 244 stderr = err.getBytes(); 245 } catch (IOException e) { 246 if (reportExceptions) { 247 LOGGER.log(Level.SEVERE, String.format("Failed to read from process: %s", cmdList.get(0)), e); 248 } 249 } catch (InterruptedException e) { 250 if (reportExceptions) { 251 LOGGER.log(Level.SEVERE, String.format("Waiting for process interrupted: %s", cmdList.get(0)), e); 252 } 253 } finally { 254 // Stop timer thread if the instance exists. 255 if (timer != null) { 256 timer.cancel(); 257 } 258 try { 259 if (process != null) { 260 IOUtils.close(process.getOutputStream()); 261 IOUtils.close(process.getInputStream()); 262 IOUtils.close(process.getErrorStream()); 263 ret = process.exitValue(); 264 } 265 } catch (IllegalThreadStateException e) { 266 process.destroy(); 267 } 268 } 269 270 if (ret != 0 && reportExceptions) { 271 int MAX_MSG_SZ = 512; /* limit to avoid flooding the logs */ 272 StringBuilder msg = new StringBuilder("Non-zero exit status ") 273 .append(ret).append(" from command [") 274 .append(cmd_str) 275 .append("] in directory ") 276 .append(dir_str); 277 if (stderr != null && stderr.length > 0) { 278 msg.append(": "); 279 if (stderr.length > MAX_MSG_SZ) { 280 msg.append(new String(stderr, 0, MAX_MSG_SZ)).append("..."); 281 } else { 282 msg.append(new String(stderr)); 283 } 284 } 285 LOGGER.log(Level.WARNING, msg.toString()); 286 } 287 288 return ret; 289 } 290 291 /** 292 * Get the output from the process as a string. 293 * 294 * @return The output from the process 295 */ getOutputString()296 public String getOutputString() { 297 String ret = null; 298 if (stdout != null) { 299 ret = new String(stdout); 300 } 301 302 return ret; 303 } 304 305 /** 306 * Get a reader to read the output from the process. 307 * 308 * @return A reader reading the process output 309 */ getOutputReader()310 public Reader getOutputReader() { 311 return new InputStreamReader(getOutputStream()); 312 } 313 314 /** 315 * Get an input stream read the output from the process. 316 * 317 * @return A reader reading the process output 318 */ getOutputStream()319 public InputStream getOutputStream() { 320 return new ByteArrayInputStream(stdout); 321 } 322 323 /** 324 * Get the output from the process written to the error stream as a string. 325 * 326 * @return The error output from the process 327 */ getErrorString()328 public String getErrorString() { 329 String ret = null; 330 if (stderr != null) { 331 ret = new String(stderr); 332 } 333 334 return ret; 335 } 336 337 /** 338 * Get a reader to read the output the process wrote to the error stream. 339 * 340 * @return A reader reading the process error stream 341 */ getErrorReader()342 public Reader getErrorReader() { 343 return new InputStreamReader(getErrorStream()); 344 } 345 346 /** 347 * Get an input stream to read the output the process wrote to the error stream. 348 * 349 * @return An input stream for reading the process error stream 350 */ getErrorStream()351 public InputStream getErrorStream() { 352 return new ByteArrayInputStream(stderr); 353 } 354 355 /** 356 * You should use the StreamHandler interface if you would like to process 357 * the output from a process while it is running. 358 */ 359 public interface StreamHandler { 360 361 /** 362 * Process the data in the stream. The processStream function is 363 * called _once_ during the lifetime of the process, and you should 364 * process all of the input you want before returning from the function. 365 * 366 * @param in The InputStream containing the data 367 * @throws java.io.IOException if any read error 368 */ processStream(InputStream in)369 void processStream(InputStream in) throws IOException; 370 } 371 372 private static class SpoolHandler implements StreamHandler { 373 374 private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 375 getBytes()376 public byte[] getBytes() { 377 return bytes.toByteArray(); 378 } 379 380 @Override processStream(InputStream input)381 public void processStream(InputStream input) throws IOException { 382 BufferedInputStream in = new BufferedInputStream(input); 383 384 byte[] buffer = new byte[8092]; 385 int len; 386 387 while ((len = in.read(buffer)) != -1) { 388 if (len > 0) { 389 bytes.write(buffer, 0, len); 390 } 391 } 392 } 393 } 394 registerErrorHandler()395 public static void registerErrorHandler() { 396 UncaughtExceptionHandler exceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); 397 if (exceptionHandler == null) { 398 LOGGER.log(Level.FINE, "Installing default uncaught exception handler"); 399 Thread.setDefaultUncaughtExceptionHandler((t, e) -> 400 LOGGER.log(Level.SEVERE, String.format("Uncaught exception in thread %s with ID %d: %s", 401 t.getName(), t.getId(), e.getMessage()), e)); 402 } 403 } 404 405 /** 406 * Build a string from the specified argv list with optional tab-indenting 407 * and line-continuations if {@code multiline} is {@code true}. 408 * @param isWindows a value indicating if the platform is Windows so that 409 * PowerShell escaping is done; else Bourne shell escaping 410 * is done. 411 * @return a defined instance 412 */ escapeForShell(List<String> argv, boolean multiline, boolean isWindows)413 public static String escapeForShell(List<String> argv, boolean multiline, boolean isWindows) { 414 StringBuilder result = new StringBuilder(); 415 for (int i = 0; i < argv.size(); ++i) { 416 if (multiline && i > 0) { 417 result.append("\t"); 418 } 419 String arg = argv.get(i); 420 result.append(isWindows ? maybeEscapeForPowerShell(arg) : maybeEscapeForSh(arg)); 421 if (i + 1 < argv.size()) { 422 if (!multiline) { 423 result.append(" "); 424 } else { 425 result.append(isWindows ? " `" : " \\"); 426 result.append(System.lineSeparator()); 427 } 428 } 429 } 430 return result.toString(); 431 } 432 maybeEscapeForSh(String value)433 private static String maybeEscapeForSh(String value) { 434 Matcher m = ARG_UNIX_QUOTING.matcher(value); 435 if (!m.find()) { 436 return value; 437 } 438 m = ARG_GNU_STYLE_EQ.matcher(value); 439 if (!m.find()) { 440 return "$'" + escapeForSh(value) + "'"; 441 } 442 String following = value.substring(m.end()); 443 return m.group() + "$'" + escapeForSh(following) + "'"; 444 } 445 escapeForSh(String value)446 private static String escapeForSh(String value) { 447 return value.replace("\\", "\\\\"). 448 replace("'", "\\'"). 449 replace("\n", "\\n"). 450 replace("\r", "\\r"). 451 replace("\f", "\\f"). 452 replace("\u0011", "\\v"). 453 replace("\t", "\\t"); 454 } 455 maybeEscapeForPowerShell(String value)456 private static String maybeEscapeForPowerShell(String value) { 457 Matcher m = ARG_WIN_QUOTING.matcher(value); 458 if (!m.find()) { 459 return value; 460 } 461 m = ARG_GNU_STYLE_EQ.matcher(value); 462 if (!m.find()) { 463 return "\"" + escapeForPowerShell(value) + "\""; 464 } 465 String following = value.substring(m.end()); 466 return m.group() + "\"" + escapeForPowerShell(following) + "\""; 467 } 468 escapeForPowerShell(String value)469 private static String escapeForPowerShell(String value) { 470 return value.replace("`", "``"). 471 replace("\"", "`\""). 472 replace("$", "`$"). 473 replace("\n", "`n"). 474 replace("\r", "`r"). 475 replace("\f", "`f"). 476 replace("\u0011", "`v"). 477 replace("\t", "`t"); 478 } 479 } 480