1#!/usr/bin/env bash 2# Licensed to the Apache Software Foundation (ASF) under one or more 3# contributor license agreements. See the NOTICE file distributed with 4# this work for additional information regarding copyright ownership. 5# The ASF licenses this file to You under the Apache License, Version 2.0 6# (the "License"); you may not use this file except in compliance with 7# the License. You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17# This is a Yetus precommit "personality" (aka customized configuration) for Lucene/Solr. 18# 19# See the Yetus precommit documentation at https://yetus.apache.org/documentation/0.7.0/ 20# and especially https://yetus.apache.org/documentation/0.7.0/precommit-advanced/. 21# See also the Yetus source code for other projects' personality examples at 22# https://git-wip-us.apache.org/repos/asf?p=yetus.git;f=precommit/personality;a=tree;hb=HEAD 23# 24# To add a new validation method (aka "plugin"): 25# 1) Add its name to the PLUGIN_LIST below 26# 2) Invoke "add_test_type" with it below 27# 3) Add a "<plugin>_filefilter" function to decide whether the plugin needs to be run based on changed files 28# 4) Add a "<plugin>_rebuild" function to call out to ant to perform the validation method. 29# See examples of the above-described function types ^^ below. 30 31# Both compile+javac plugins are required, as well as unit+junit: in both cases, neither work individually 32PLUGIN_LIST="ant,jira,compile,javac,unit,junit,test4tests" 33PLUGIN_LIST+=",testoutput,checkluceneversion,ratsources,checkforbiddenapis,checklicenses" 34PLUGIN_LIST+=",validatesourcepatterns,validaterefguide" 35personality_plugins "${PLUGIN_LIST}" 36 37add_test_type "checkluceneversion" 38add_test_type "ratsources" 39add_test_type "checkforbiddenapis" 40add_test_type "checklicenses" 41add_test_type "validatesourcepatterns" 42add_test_type "validaterefguide" 43 44add_test_format "testoutput" 45 46## @description Globals specific to this personality 47## @audience private 48## @stability evolving 49function personality_globals 50{ 51 #shellcheck disable=SC2034 52 PATCH_BRANCH_DEFAULT=master 53 #shellcheck disable=SC2034 54 JIRA_ISSUE_RE='^(LUCENE|SOLR)-[0-9]+$' 55 #shellcheck disable=SC2034 56 JIRA_STATUS_RE='Patch Available' 57 #shellcheck disable=SC2034 58 GITHUB_REPO="apache/lucene-solr" 59 #shellcheck disable=SC2034 60 BUILDTOOL=ant 61} 62 63## @description Queue up modules for this personality 64## @audience private 65## @stability evolving 66## @param repostatus 67## @param testtype 68function personality_modules 69{ 70 local repostatus=$1 71 local testtype=$2 72 73 local module 74 local extra 75 76 local moduleType="submodules" 77 78 yetus_debug "Personality (lucene-solr): ${repostatus} ${testtype}" 79 80 clear_personality_queue 81 82 case ${testtype} in 83 clean|distclean|validatesourcepatterns) 84 moduleType="top" 85 ;; 86 checkluceneversion) 87 moduleType="solr" 88 ;; 89 ratsources) 90 moduleType="submodules" 91 ;; 92 checkforbiddenapis) 93 moduleType="both" 94 ;; 95 checklicenses) 96 moduleType="mains" 97 ;; 98 validaterefguide) 99 moduleType="solr-ref-guide" 100 ;; 101 compile) 102 moduleType="submodules" 103 extra="compile-test" 104 ;; 105 junit|unit) 106 moduleType="submodules" 107 extra="test" 108 ;; 109 *) 110 ;; 111 esac 112 113 case ${moduleType} in 114 submodules) 115 for module in "${CHANGED_MODULES[@]}"; do 116 if [[ ! "${module}" =~ ^lucene/(licenses|site) ]]; then # blacklist lucene/ dirs that aren't modules 117 if [[ "${module}" =~ ^(lucene/(analysis/[^/]+|[^/]+)) ]]; then 118 local lucene_module=${BASH_REMATCH[0]} 119 personality_enqueue_module "${lucene_module}" "${extra}" 120 elif [[ "${module}" =~ ^solr/(core|solrj|test-framework|solr-ref-guide|contrib/[^.]+) ]]; then # whitelist solr/ modules 121 local solr_module=${BASH_REMATCH[0]} 122 # In solr-ref-guide module, do not execute "compile" or "unit" plugins 123 if [[ ! "${solr_module}" == solr/solr-ref-guide || ! ${testtype} =~ ^(compile|unit)$ ]]; then 124 personality_enqueue_module "${solr_module}" "${extra}" 125 fi 126 fi 127 fi 128 done 129 ;; 130 lucene|solr) 131 personality_enqueue_module "${moduleType}" "${extra}" 132 ;; 133 top) 134 personality_enqueue_module . "${extra}" 135 ;; 136 mains) 137 personality_enqueue_module "lucene" "${extra}" 138 personality_enqueue_module "solr" "${extra}" 139 ;; 140 both) # solr, lucene, or both 141 # personality_enqueue_module KEEPS duplicates, so de-dupe first 142 local doSolr=0,doLucene=0 143 for module in "${CHANGED_MODULES[@]}"; do 144 if [[ "${module}" =~ ^solr/ ]]; then doSolr=1; fi 145 if [[ "${module}" =~ ^lucene/ ]]; then doLucene=1; fi 146 done 147 if [[ ${doLucene} == 1 ]]; then 148 if [[ ${doSolr} == 1 ]]; then 149 personality_enqueue_module . "${extra}" 150 else 151 personality_enqueue_module "lucene" "${extra}" 152 fi 153 elif [[ ${doSolr} == 1 ]]; then 154 personality_enqueue_module "solr" "${extra}" 155 fi 156 ;; 157 solr-ref-guide) 158 for module in "${CHANGED_MODULES[@]}"; do 159 if [[ "${module}" =~ ^solr/solr-ref-guide ]]; then 160 personality_enqueue_module "solr/solr-ref-guide" "${extra}" 161 fi 162 done 163 ;; 164 *) 165 ;; 166 esac 167} 168 169## @description Add tests based upon personality needs 170## @audience private 171## @stability evolving 172## @param filename 173function personality_file_tests 174{ 175 declare filename=$1 176 177 yetus_debug "Using Lucene/Solr-specific personality_file_tests" 178 179 if [[ ! ${filename} =~ solr-ref-guide ]]; then 180 if [[ ${filename} =~ build\.xml$ || ${filename} =~ /src/(java|resources|test|test-files|tools) ]]; then 181 yetus_debug "tests/unit: ${filename}" 182 add_test compile 183 add_test javac 184 add_test unit 185 fi 186 fi 187} 188 189## @description hook to reroute junit folder to search test results based on the module 190## @audience private 191## @stability evolving 192## @param module 193## @param buildlogfile 194function testoutput_process_tests 195{ 196 # shellcheck disable=SC2034 197 declare module=$1 198 declare buildlogfile=$2 199 if [[ "${module}" =~ ^lucene/analysis/ ]]; then 200 JUNIT_TEST_OUTPUT_DIR="../../build/${module#*/}" 201 elif [[ "${module}" =~ ^lucene/ ]]; then 202 JUNIT_TEST_OUTPUT_DIR="../build/${module#*/}" 203 elif [[ "${module}" =~ ^solr/contrib/extraction ]]; then 204 JUNIT_TEST_OUTPUT_DIR="../../build/contrib/solr-extraction" 205 elif [[ "${module}" =~ ^solr/contrib/(.*) ]]; then 206 JUNIT_TEST_OUTPUT_DIR="../../build/contrib/solr-${BASH_REMATCH[1]}" 207 elif [[ "${module}" =~ ^solr/(.*) ]]; then 208 JUNIT_TEST_OUTPUT_DIR="../build/solr-${BASH_REMATCH[1]}" 209 fi 210 yetus_debug "Rerouting build dir for junit to ${JUNIT_TEST_OUTPUT_DIR}" 211} 212 213## @description checkluceneversion file filter 214## @audience private 215## @stability evolving 216## @param filename 217function checkluceneversion_filefilter 218{ 219 local filename=$1 220 if [[ ${filename} =~ ^solr/(example|server/solr/configsets) ]]; then 221 yetus_debug "tests/checkluceneversion: ${filename}" 222 add_test checkluceneversion 223 fi 224} 225 226## @description checkluceneversion test 227## @audience private 228## @stability evolving 229## @param repostatus 230function checkluceneversion_rebuild 231{ 232 local repostatus=$1 233 lucene_ant_command ${repostatus} "checkluceneversion" "check-example-lucene-match-version" "Check configsets' lucene version" 234} 235 236## @description ratsources file filter 237## @audience private 238## @stability evolving 239## @param filename 240function ratsources_filefilter 241{ 242 local filename=$1 243 if [[ ${filename} =~ /src/|\.xml$ ]] ; then 244 yetus_debug "tests/ratsources: ${filename}" 245 add_test ratsources 246 fi 247} 248 249## @description ratsources test 250## @audience private 251## @stability evolving 252## @param repostatus 253function ratsources_rebuild 254{ 255 local repostatus=$1 256 lucene_ant_command ${repostatus} "ratsources" "rat-sources" "Release audit (RAT)" 257} 258 259## @description checkforbiddenapis file filter 260## @audience private 261## @stability evolving 262## @param filename 263function checkforbiddenapis_filefilter 264{ 265 local filename=$1 266 if [[ ${filename} =~ \.java$ ]] ; then 267 yetus_debug "tests/checkforbiddenapis: ${filename}" 268 add_test checkforbiddenapis 269 fi 270} 271 272## @description checkforbiddenapis test 273## @audience private 274## @stability evolving 275## @param repostatus 276function checkforbiddenapis_rebuild 277{ 278 local repostatus=$1 279 lucene_ant_command ${repostatus} "checkforbiddenapis" "check-forbidden-apis" "Check forbidden APIs" 280} 281 282## @description checklicenses file filter 283## @audience private 284## @stability evolving 285## @param filename 286function checklicenses_filefilter 287{ 288 local filename=$1 289 if [[ ${filename} =~ (lucene|solr)/licenses/|lucene/ivy-versions.properties$ ]]; then 290 yetus_debug "tests/checklicenses: ${filename}" 291 add_test checklicenses 292 fi 293} 294 295## @description checklicenses test 296## @audience private 297## @stability evolving 298## @param repostatus 299function checklicenses_rebuild 300{ 301 local repostatus=$1 302 lucene_ant_command ${repostatus} "checklicenses" "check-licenses" "Check licenses" 303} 304 305## @description validaterefguide file filter 306## @audience private 307## @stability evolving 308## @param filename 309function validaterefguide_filefilter 310{ 311 local filename=$1 312 if [[ ${filename} =~ solr/solr-ref-guide ]]; then 313 yetus_debug "tests/validaterefguide: ${filename}" 314 add_test validaterefguide 315 fi 316} 317 318## @description validaterefguide test 319## @audience private 320## @stability evolving 321## @param repostatus 322function validaterefguide_rebuild 323{ 324 local repostatus=$1 325 lucene_ant_command ${repostatus} "validaterefguide" "bare-bones-html-validation" "Validate ref guide" 326} 327 328## @description validatesourcepatterns file filter 329## @audience private 330## @stability evolving 331## @param filename 332function validatesourcepatterns_filefilter 333{ 334 local filename=$1 335 if [[ ${filename} =~ \.(java|jflex|py|pl|g4|jj|html|js|css|xml|xsl|vm|sh|cmd|bat|policy|properties|mdtext|groovy|template|adoc|json)$ ]]; then 336 yetus_debug "tests/validatesourcepatterns: ${filename}" 337 add_test validatesourcepatterns 338 fi 339} 340 341## @description validatesourcepatterns test 342## @audience private 343## @stability evolving 344## @param repostatus 345function validatesourcepatterns_rebuild 346{ 347 local repostatus=$1 348 lucene_ant_command ${repostatus} "validatesourcepatterns" "validate-source-patterns" "Validate source patterns" 349} 350 351function lucene_ant_command 352{ 353 declare repostatus=$1 354 declare testname=$2 355 declare antcommand=$3 356 declare title=$4 357 358 declare result=0 359 declare i=0 360 declare module 361 declare fn 362 declare result 363 364 if [[ "${repostatus}" = branch ]]; then 365 return 0 366 fi 367 368 if ! verify_needed_test ${testname}; then 369 echo "${BUILDMODEMSG} does not need ${testname} testing." 370 return 0 371 fi 372 373 big_console_header "${title}" 374 personality_modules ${repostatus} ${testname} 375 until [[ ${i} -eq ${#MODULE[@]} ]]; do 376 if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then 377 ((result=result+1)) 378 ((i=i+1)) 379 continue 380 fi 381 ANT_ARGS=${antcommand} 382 383 start_clock 384 module=${MODULE[${i}]} 385 fn=$(module_file_fragment "${module}") 386 logfilename="${repostatus}-${antcommand}-${fn}.txt"; 387 logfile="${PATCH_DIR}/${logfilename}" 388 buildtool_cwd "${i}" 389 echo_and_redirect "${logfile}" $(ant_executor) 390 391 if [[ $? == 0 ]] ; then 392 module_status ${i} +1 "${logfilename}" "${title}" "${antcommand} passed" 393 else 394 module_status ${i} -1 "${logfilename}" "${title}" "${antcommand} failed" 395 ((result=result+1)) 396 fi 397 ((i=i+1)) 398 savestop=$(stop_clock) 399 MODULE_STATUS_TIMER[${i}]=${savestop} 400 done 401 ANT_ARGS="" 402 if [[ ${result} -gt 0 ]]; then 403 modules_messages ${repostatus} "${title}" false 404 return 1 405 fi 406 modules_messages ${repostatus} "${title}" true 407 return 0 408} 409