xref: /Lucene/dev-tools/scripts/smokeTestRelease.py (revision 10a43d916e7d282cf98f35305c411fc9060004ab) 
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Licensed to the Apache Software Foundation (ASF) under one or more
4# contributor license agreements.  See the NOTICE file distributed with
5# this work for additional information regarding copyright ownership.
6# The ASF licenses this file to You under the Apache License, Version 2.0
7# (the "License"); you may not use this file except in compliance with
8# the License.  You may obtain a copy of the License at
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17
18import argparse
19import codecs
20import datetime
21import filecmp
22import glob
23import hashlib
24import http.client
25import os
26import platform
27import re
28import shutil
29import subprocess
30import sys
31import textwrap
32import traceback
33import urllib.error
34import urllib.parse
35import urllib.parse
36import urllib.request
37import xml.etree.ElementTree as ET
38import zipfile
39from collections import namedtuple
40import scriptutil
41
42# This tool expects to find /lucene off the base URL.  You
43# must have a working gpg, tar, unzip in your path.  This has been
44# tested on Linux and on Cygwin under Windows 7.
45
46cygwin = platform.system().lower().startswith('cygwin')
47cygwinWindowsRoot = os.popen('cygpath -w /').read().strip().replace('\\','/') if cygwin else ''
48
49
50def unshortenURL(url):
51  parsed = urllib.parse.urlparse(url)
52  if parsed[0] in ('http', 'https'):
53    h = http.client.HTTPConnection(parsed.netloc)
54    h.request('HEAD', parsed.path)
55    response = h.getresponse()
56    if int(response.status/100) == 3 and response.getheader('Location'):
57      return response.getheader('Location')
58  return url
59
60# TODO
61#   - make sure jars exist inside bin release
62#   - make sure docs exist
63
64reHREF = re.compile('<a href="(.*?)">(.*?)</a>')
65
66# Set to False to avoid re-downloading the packages...
67FORCE_CLEAN = True
68
69
70def getHREFs(urlString):
71
72  # Deref any redirects
73  while True:
74    url = urllib.parse.urlparse(urlString)
75    if url.scheme == "http":
76      h = http.client.HTTPConnection(url.netloc)
77    elif url.scheme == "https":
78      h = http.client.HTTPSConnection(url.netloc)
79    else:
80      raise RuntimeError("Unknown protocol: %s" % url.scheme)
81    h.request('HEAD', url.path)
82    r = h.getresponse()
83    newLoc = r.getheader('location')
84    if newLoc is not None:
85      urlString = newLoc
86    else:
87      break
88
89  links = []
90  try:
91    html = load(urlString)
92  except:
93    print('\nFAILED to open url %s' % urlString)
94    traceback.print_exc()
95    raise
96
97  for subUrl, text in reHREF.findall(html):
98    fullURL = urllib.parse.urljoin(urlString, subUrl)
99    links.append((text, fullURL))
100  return links
101
102
103def load(urlString):
104  try:
105    content = urllib.request.urlopen(urlString).read().decode('utf-8')
106  except Exception as e:
107    print('Retrying download of url %s after exception: %s' % (urlString, e))
108    content = urllib.request.urlopen(urlString).read().decode('utf-8')
109  return content
110
111
112def noJavaPackageClasses(desc, file):
113  with zipfile.ZipFile(file) as z2:
114    for name2 in z2.namelist():
115      if name2.endswith('.class') and (name2.startswith('java/') or name2.startswith('javax/')):
116        raise RuntimeError('%s contains sheisty class "%s"' % (desc, name2))
117
118
119def decodeUTF8(bytes):
120  return codecs.getdecoder('UTF-8')(bytes)[0]
121
122
123MANIFEST_FILE_NAME = 'META-INF/MANIFEST.MF'
124NOTICE_FILE_NAME = 'META-INF/NOTICE.txt'
125LICENSE_FILE_NAME = 'META-INF/LICENSE.txt'
126
127
128def checkJARMetaData(desc, jarFile, gitRevision, version):
129
130  with zipfile.ZipFile(jarFile, 'r') as z:
131    for name in (MANIFEST_FILE_NAME, NOTICE_FILE_NAME, LICENSE_FILE_NAME):
132      try:
133        # The Python docs state a KeyError is raised ... so this None
134        # check is just defensive:
135        if z.getinfo(name) is None:
136          raise RuntimeError('%s is missing %s' % (desc, name))
137      except KeyError:
138        raise RuntimeError('%s is missing %s' % (desc, name))
139
140    s = decodeUTF8(z.read(MANIFEST_FILE_NAME))
141
142    for verify in (
143      'Specification-Vendor: The Apache Software Foundation',
144      'Implementation-Vendor: The Apache Software Foundation',
145      'Specification-Title: Lucene Search Engine:',
146      'Implementation-Title: org.apache.lucene',
147      'X-Compile-Source-JDK: 17',
148      'X-Compile-Target-JDK: 17',
149      'Specification-Version: %s' % version,
150      'X-Build-JDK: 17.',
151      'Extension-Name: org.apache.lucene'):
152      if type(verify) is not tuple:
153        verify = (verify,)
154      for x in verify:
155        if s.find(x) != -1:
156          break
157      else:
158        if len(verify) == 1:
159          raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF' % (desc, verify[0]))
160        else:
161          raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF' % (desc, verify))
162
163    if gitRevision != 'skip':
164      # Make sure this matches the version and git revision we think we are releasing:
165      match = re.search("Implementation-Version: (.+\r\n .+)", s, re.MULTILINE)
166      if match:
167        implLine = match.group(1).replace("\r\n ", "")
168        verifyRevision = '%s %s' % (version, gitRevision)
169        if implLine.find(verifyRevision) == -1:
170          raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF (wrong git revision?)' % \
171                           (desc, verifyRevision))
172      else:
173        raise RuntimeError('%s is missing Implementation-Version inside its META-INF/MANIFEST.MF' % desc)
174
175    notice = decodeUTF8(z.read(NOTICE_FILE_NAME))
176    lucene_license = decodeUTF8(z.read(LICENSE_FILE_NAME))
177
178    if LUCENE_LICENSE is None:
179      raise RuntimeError('BUG in smokeTestRelease!')
180    if LUCENE_NOTICE is None:
181      raise RuntimeError('BUG in smokeTestRelease!')
182    if notice != LUCENE_NOTICE:
183      raise RuntimeError('%s: %s contents doesn\'t match main NOTICE.txt' % \
184                         (desc, NOTICE_FILE_NAME))
185    if lucene_license != LUCENE_LICENSE:
186      raise RuntimeError('%s: %s contents doesn\'t match main LICENSE.txt' % \
187                         (desc, LICENSE_FILE_NAME))
188
189
190def normSlashes(path):
191  return path.replace(os.sep, '/')
192
193
194def checkAllJARs(topDir, gitRevision, version):
195  print('    verify JAR metadata/identity/no javax.* or java.* classes...')
196  for root, dirs, files in os.walk(topDir):
197
198    normRoot = normSlashes(root)
199
200    for file in files:
201      if file.lower().endswith('.jar'):
202        if normRoot.endswith('/replicator/lib') and file.startswith('javax.servlet'):
203          continue
204        fullPath = '%s/%s' % (root, file)
205        noJavaPackageClasses('JAR file "%s"' % fullPath, fullPath)
206        if file.lower().find('lucene') != -1:
207          checkJARMetaData('JAR file "%s"' % fullPath, fullPath, gitRevision, version)
208
209
210def checkSigs(urlString, version, tmpDir, isSigned, keysFile):
211  print('  test basics...')
212  ents = getDirEntries(urlString)
213  artifact = None
214  changesURL = None
215  mavenURL = None
216  artifactURL = None
217  expectedSigs = []
218  if isSigned:
219    expectedSigs.append('asc')
220  expectedSigs.extend(['sha512'])
221  sigs = []
222  artifacts = []
223
224  for text, subURL in ents:
225    if text == 'KEYS':
226      raise RuntimeError('lucene: release dir should not contain a KEYS file - only toplevel /dist/lucene/KEYS is used')
227    elif text == 'maven/':
228      mavenURL = subURL
229    elif text.startswith('changes'):
230      if text not in ('changes/', 'changes-%s/' % version):
231        raise RuntimeError('lucene: found %s vs expected changes-%s/' % (text, version))
232      changesURL = subURL
233    elif artifact is None:
234      artifact = text
235      artifactURL = subURL
236      expected = 'lucene-%s' % version
237      if not artifact.startswith(expected):
238        raise RuntimeError('lucene: unknown artifact %s: expected prefix %s' % (text, expected))
239      sigs = []
240    elif text.startswith(artifact + '.'):
241      sigs.append(text[len(artifact)+1:])
242    else:
243      if sigs != expectedSigs:
244        raise RuntimeError('lucene: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs))
245      artifacts.append((artifact, artifactURL))
246      artifact = text
247      artifactURL = subURL
248      sigs = []
249
250  if sigs != []:
251    artifacts.append((artifact, artifactURL))
252    if sigs != expectedSigs:
253      raise RuntimeError('lucene: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs))
254
255  expected = ['lucene-%s-src.tgz' % version,
256              'lucene-%s.tgz' % version]
257
258  actual = [x[0] for x in artifacts]
259  if expected != actual:
260    raise RuntimeError('lucene: wrong artifacts: expected %s but got %s' % (expected, actual))
261
262  # Set up clean gpg world; import keys file:
263  gpgHomeDir = '%s/lucene.gpg' % tmpDir
264  if os.path.exists(gpgHomeDir):
265    shutil.rmtree(gpgHomeDir)
266  os.makedirs(gpgHomeDir, 0o700)
267  run('gpg --homedir %s --import %s' % (gpgHomeDir, keysFile),
268      '%s/lucene.gpg.import.log' % tmpDir)
269
270  if mavenURL is None:
271    raise RuntimeError('lucene is missing maven')
272
273  if changesURL is None:
274    raise RuntimeError('lucene is missing changes-%s' % version)
275  testChanges(version, changesURL)
276
277  for artifact, urlString in artifacts:
278    print('  download %s...' % artifact)
279    scriptutil.download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN)
280    verifyDigests(artifact, urlString, tmpDir)
281
282    if isSigned:
283      print('    verify sig')
284      # Test sig (this is done with a clean brand-new GPG world)
285      scriptutil.download(artifact + '.asc', urlString + '.asc', tmpDir, force_clean=FORCE_CLEAN)
286      sigFile = '%s/%s.asc' % (tmpDir, artifact)
287      artifactFile = '%s/%s' % (tmpDir, artifact)
288      logFile = '%s/lucene.%s.gpg.verify.log' % (tmpDir, artifact)
289      run('gpg --homedir %s --display-charset utf-8 --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
290          logFile)
291      # Forward any GPG warnings, except the expected one (since it's a clean world)
292      with open(logFile) as f:
293        print("File: %s" % logFile)
294        for line in f.readlines():
295          if line.lower().find('warning') != -1 \
296            and line.find('WARNING: This key is not certified with a trusted signature') == -1:
297              print('      GPG: %s' % line.strip())
298
299      # Test trust (this is done with the real users config)
300      run('gpg --import %s' % (keysFile),
301          '%s/lucene.gpg.trust.import.log' % tmpDir)
302      print('    verify trust')
303      logFile = '%s/lucene.%s.gpg.trust.log' % (tmpDir, artifact)
304      run('gpg --display-charset utf-8 --verify %s %s' % (sigFile, artifactFile), logFile)
305      # Forward any GPG warnings:
306      with open(logFile) as f:
307        for line in f.readlines():
308          if line.lower().find('warning') != -1:
309            print('      GPG: %s' % line.strip())
310
311
312def testChanges(version, changesURLString):
313  print('  check changes HTML...')
314  changesURL = None
315  for text, subURL in getDirEntries(changesURLString):
316    if text == 'Changes.html':
317      changesURL = subURL
318
319  if changesURL is None:
320    raise RuntimeError('did not see Changes.html link from %s' % changesURLString)
321
322  s = load(changesURL)
323  checkChangesContent(s, version, changesURL, True)
324
325
326def testChangesText(dir, version):
327  "Checks all CHANGES.txt under this dir."
328  for root, dirs, files in os.walk(dir):
329
330    # NOTE: O(N) but N should be smallish:
331    if 'CHANGES.txt' in files:
332      fullPath = '%s/CHANGES.txt' % root
333      #print 'CHECK %s' % fullPath
334      checkChangesContent(open(fullPath, encoding='UTF-8').read(), version, fullPath, False)
335
336reChangesSectionHREF = re.compile('<a id="(.*?)".*?>(.*?)</a>', re.IGNORECASE)
337reUnderbarNotDashHTML = re.compile(r'<li>(\s*(LUCENE)_\d\d\d\d+)')
338reUnderbarNotDashTXT = re.compile(r'\s+((LUCENE)_\d\d\d\d+)', re.MULTILINE)
339
340
341def checkChangesContent(s, version, name, isHTML):
342  currentVersionTuple = versionToTuple(version, name)
343
344  if isHTML and s.find('Release %s' % version) == -1:
345    raise RuntimeError('did not see "Release %s" in %s' % (version, name))
346
347  if isHTML:
348    r = reUnderbarNotDashHTML
349  else:
350    r = reUnderbarNotDashTXT
351
352  m = r.search(s)
353  if m is not None:
354    raise RuntimeError('incorrect issue (_ instead of -) in %s: %s' % (name, m.group(1)))
355
356  if s.lower().find('not yet released') != -1:
357    raise RuntimeError('saw "not yet released" in %s' % name)
358
359  if not isHTML:
360    sub = 'Lucene %s' % version
361    if s.find(sub) == -1:
362      # benchmark never seems to include release info:
363      if name.find('/benchmark/') == -1:
364        raise RuntimeError('did not see "%s" in %s' % (sub, name))
365
366  if isHTML:
367    # Make sure that a section only appears once under each release,
368    # and that each release is not greater than the current version
369    seenIDs = set()
370    seenText = set()
371
372    release = None
373    for id, text in reChangesSectionHREF.findall(s):
374      if text.lower().startswith('release '):
375        release = text[8:].strip()
376        seenText.clear()
377        releaseTuple = versionToTuple(release, name)
378        if releaseTuple > currentVersionTuple:
379          raise RuntimeError('Future release %s is greater than %s in %s' % (release, version, name))
380      if id in seenIDs:
381        raise RuntimeError('%s has duplicate section "%s" under release "%s"' % (name, text, release))
382      seenIDs.add(id)
383      if text in seenText:
384        raise RuntimeError('%s has duplicate section "%s" under release "%s"' % (name, text, release))
385      seenText.add(text)
386
387
388reVersion = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?\s*(-alpha|-beta|final|RC\d+)?\s*(?:\[.*\])?', re.IGNORECASE)
389
390
391def versionToTuple(version, name):
392  versionMatch = reVersion.match(version)
393  if versionMatch is None:
394    raise RuntimeError('Version %s in %s cannot be parsed' % (version, name))
395  versionTuple = versionMatch.groups()
396  while versionTuple[-1] is None or versionTuple[-1] == '':
397    versionTuple = versionTuple[:-1]
398  if versionTuple[-1].lower() == '-alpha':
399    versionTuple = versionTuple[:-1] + ('0',)
400  elif versionTuple[-1].lower() == '-beta':
401    versionTuple = versionTuple[:-1] + ('1',)
402  elif versionTuple[-1].lower() == 'final':
403    versionTuple = versionTuple[:-2] + ('100',)
404  elif versionTuple[-1].lower()[:2] == 'rc':
405    versionTuple = versionTuple[:-2] + (versionTuple[-1][2:],)
406  return tuple(int(x) if x is not None and x.isnumeric() else x for x in versionTuple)
407
408
409reUnixPath = re.compile(r'\b[a-zA-Z_]+=(?:"(?:\\"|[^"])*"' + '|(?:\\\\.|[^"\'\\s])*' + r"|'(?:\\'|[^'])*')" \
410                        + r'|(/(?:\\.|[^"\'\s])*)' \
411                        + r'|("/(?:\\.|[^"])*")'   \
412                        + r"|('/(?:\\.|[^'])*')")
413
414
415def unix2win(matchobj):
416  if matchobj.group(1) is not None: return cygwinWindowsRoot + matchobj.group()
417  if matchobj.group(2) is not None: return '"%s%s' % (cygwinWindowsRoot, matchobj.group().lstrip('"'))
418  if matchobj.group(3) is not None: return "'%s%s" % (cygwinWindowsRoot, matchobj.group().lstrip("'"))
419  return matchobj.group()
420
421
422def cygwinifyPaths(command):
423  # The problem: Native Windows applications running under Cygwin can't
424  # handle Cygwin's Unix-style paths.  However, environment variable
425  # values are automatically converted, so only paths outside of
426  # environment variable values should be converted to Windows paths.
427  # Assumption: all paths will be absolute.
428  if '; gradlew ' in command: command = reUnixPath.sub(unix2win, command)
429  return command
430
431
432def printFileContents(fileName):
433
434  # Assume log file was written in system's default encoding, but
435  # even if we are wrong, we replace errors ... the ASCII chars
436  # (which is what we mostly care about eg for the test seed) should
437  # still survive:
438  txt = codecs.open(fileName, 'r', encoding=sys.getdefaultencoding(), errors='replace').read()
439
440  # Encode to our output encoding (likely also system's default
441  # encoding):
442  bytes = txt.encode(sys.stdout.encoding, errors='replace')
443
444  # Decode back to string and print... we should hit no exception here
445  # since all errors have been replaced:
446  print(codecs.getdecoder(sys.stdout.encoding)(bytes)[0])
447  print()
448
449
450def run(command, logFile):
451  if cygwin: command = cygwinifyPaths(command)
452  if os.system('%s > %s 2>&1' % (command, logFile)):
453    logPath = os.path.abspath(logFile)
454    print('\ncommand "%s" failed:' % command)
455    printFileContents(logFile)
456    raise RuntimeError('command "%s" failed; see log file %s' % (command, logPath))
457
458
459def verifyDigests(artifact, urlString, tmpDir):
460  print('    verify sha512 digest')
461  sha512Expected, t = load(urlString + '.sha512').strip().split()
462  if t != '*'+artifact:
463    raise RuntimeError('SHA512 %s.sha512 lists artifact %s but expected *%s' % (urlString, t, artifact))
464
465  s512 = hashlib.sha512()
466  f = open('%s/%s' % (tmpDir, artifact), 'rb')
467  while True:
468    x = f.read(65536)
469    if len(x) == 0:
470      break
471    s512.update(x)
472  f.close()
473  sha512Actual = s512.hexdigest()
474  if sha512Actual != sha512Expected:
475    raise RuntimeError('SHA512 digest mismatch for %s: expected %s but got %s' % (artifact, sha512Expected, sha512Actual))
476
477
478def getDirEntries(urlString):
479  if urlString.startswith('file:/') and not urlString.startswith('file://'):
480    # stupid bogus ant URI
481    urlString = "file:///" + urlString[6:]
482
483  if urlString.startswith('file://'):
484    path = urlString[7:]
485    if path.endswith('/'):
486      path = path[:-1]
487    if cygwin: # Convert Windows path to Cygwin path
488      path = re.sub(r'^/([A-Za-z]):/', r'/cygdrive/\1/', path)
489    l = []
490    for ent in os.listdir(path):
491      entPath = '%s/%s' % (path, ent)
492      if os.path.isdir(entPath):
493        entPath += '/'
494        ent += '/'
495      l.append((ent, 'file://%s' % entPath))
496    l.sort()
497    return l
498  else:
499    links = getHREFs(urlString)
500    for i, (text, subURL) in enumerate(links):
501      if text == 'Parent Directory' or text == '..':
502        return links[(i+1):]
503
504
505def unpackAndVerify(java, tmpDir, artifact, gitRevision, version, testArgs):
506  destDir = '%s/unpack' % tmpDir
507  if os.path.exists(destDir):
508    shutil.rmtree(destDir)
509  os.makedirs(destDir)
510  os.chdir(destDir)
511  print('  unpack %s...' % artifact)
512  unpackLogFile = '%s/lucene-unpack-%s.log' % (tmpDir, artifact)
513  if artifact.endswith('.tar.gz') or artifact.endswith('.tgz'):
514    run('tar xzf %s/%s' % (tmpDir, artifact), unpackLogFile)
515  elif artifact.endswith('.zip'):
516    run('unzip %s/%s' % (tmpDir, artifact), unpackLogFile)
517
518  # make sure it unpacks to proper subdir
519  l = os.listdir(destDir)
520  expected = 'lucene-%s' % version
521  if l != [expected]:
522    raise RuntimeError('unpack produced entries %s; expected only %s' % (l, expected))
523
524  unpackPath = '%s/%s' % (destDir, expected)
525  verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs)
526  return unpackPath
527
528LUCENE_NOTICE = None
529LUCENE_LICENSE = None
530
531
532def is_in_list(in_folder, files, indent=4):
533  for fileName in files:
534    print("%sChecking %s" % (" "*indent, fileName))
535    found = False
536    for f in [fileName, fileName + '.txt', fileName + '.md']:
537      if f in in_folder:
538        in_folder.remove(f)
539        found = True
540    if not found:
541      raise RuntimeError('file "%s" is missing' % fileName)
542
543
544def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
545  global LUCENE_NOTICE
546  global LUCENE_LICENSE
547
548  os.chdir(unpackPath)
549  isSrc = artifact.find('-src') != -1
550
551  # Check text files in release
552  print("  %s" % artifact)
553  in_root_folder = list(filter(lambda x: x[0] != '.', os.listdir(unpackPath)))
554  in_lucene_folder = []
555  if isSrc:
556    in_lucene_folder.extend(os.listdir(os.path.join(unpackPath, 'lucene')))
557    is_in_list(in_root_folder, ['LICENSE', 'NOTICE', 'README'])
558    is_in_list(in_lucene_folder, ['JRE_VERSION_MIGRATION', 'CHANGES', 'MIGRATE', 'SYSTEM_REQUIREMENTS'])
559  else:
560    is_in_list(in_root_folder, ['LICENSE', 'NOTICE', 'README', 'JRE_VERSION_MIGRATION', 'CHANGES',
561                                'MIGRATE', 'SYSTEM_REQUIREMENTS'])
562
563  if LUCENE_NOTICE is None:
564    LUCENE_NOTICE = open('%s/NOTICE.txt' % unpackPath, encoding='UTF-8').read()
565  if LUCENE_LICENSE is None:
566    LUCENE_LICENSE = open('%s/LICENSE.txt' % unpackPath, encoding='UTF-8').read()
567
568  # if not isSrc:
569  #   # TODO: we should add verifyModule/verifySubmodule (e.g. analysis) here and recurse through
570  #   expectedJARs = ()
571  #
572  #   for fileName in expectedJARs:
573  #     fileName += '.jar'
574  #     if fileName not in l:
575  #       raise RuntimeError('lucene: file "%s" is missing from artifact %s' % (fileName, artifact))
576  #     in_root_folder.remove(fileName)
577
578  expected_folders = ['analysis', 'analysis.tests', 'backward-codecs', 'benchmark', 'classification', 'codecs', 'core', 'core.tests',
579                      'distribution.tests', 'demo', 'expressions', 'facet', 'grouping', 'highlighter', 'join',
580                      'luke', 'memory', 'misc', 'monitor', 'queries', 'queryparser', 'replicator',
581                      'sandbox', 'spatial-extras', 'spatial-test-fixtures', 'spatial3d', 'suggest', 'test-framework', 'licenses']
582  if isSrc:
583    expected_src_root_files = ['build.gradle', 'buildSrc', 'CONTRIBUTING.md', 'dev-docs', 'dev-tools', 'gradle', 'gradlew',
584                               'gradlew.bat', 'help', 'lucene', 'settings.gradle', 'versions.lock', 'versions.props']
585    expected_src_lucene_files = ['build.gradle', 'documentation', 'distribution', 'dev-docs']
586    is_in_list(in_root_folder, expected_src_root_files)
587    is_in_list(in_lucene_folder, expected_folders)
588    is_in_list(in_lucene_folder, expected_src_lucene_files)
589    if len(in_lucene_folder) > 0:
590      raise RuntimeError('lucene: unexpected files/dirs in artifact %s lucene/ folder: %s' % (artifact, in_lucene_folder))
591  else:
592    is_in_list(in_root_folder, ['bin', 'docs', 'licenses', 'modules', 'modules-thirdparty', 'modules-test-framework'])
593
594  if len(in_root_folder) > 0:
595    raise RuntimeError('lucene: unexpected files/dirs in artifact %s: %s' % (artifact, in_root_folder))
596
597  if isSrc:
598    print('    make sure no JARs/WARs in src dist...')
599    lines = os.popen('find . -name \\*.jar').readlines()
600    if len(lines) != 0:
601      print('    FAILED:')
602      for line in lines:
603        print('      %s' % line.strip())
604      raise RuntimeError('source release has JARs...')
605    lines = os.popen('find . -name \\*.war').readlines()
606    if len(lines) != 0:
607      print('    FAILED:')
608      for line in lines:
609        print('      %s' % line.strip())
610      raise RuntimeError('source release has WARs...')
611
612    validateCmd = './gradlew --no-daemon check -p lucene/documentation'
613    print('    run "%s"' % validateCmd)
614    java.run_java17(validateCmd, '%s/validate.log' % unpackPath)
615
616    print("    run tests w/ Java 17 and testArgs='%s'..." % testArgs)
617    java.run_java17('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
618    print("    compile jars w/ Java 17")
619    java.run_java17('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
620    testDemo(java.run_java17, isSrc, version, '17')
621
622    if java.run_java18:
623      print("    run tests w/ Java 18 and testArgs='%s'..." % testArgs)
624      java.run_java18('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
625      print("    compile jars w/ Java 18")
626      java.run_java18('./gradlew --no-daemon jar -Dversion.release=%s' % version, '%s/compile.log' % unpackPath)
627      testDemo(java.run_java18, isSrc, version, '18')
628
629    print('  confirm all releases have coverage in TestBackwardsCompatibility')
630    confirmAllReleasesAreTestedForBackCompat(version, unpackPath)
631
632  else:
633
634    checkAllJARs(os.getcwd(), gitRevision, version)
635
636    testDemo(java.run_java17, isSrc, version, '17')
637    if java.run_java18:
638      testDemo(java.run_java18, isSrc, version, '18')
639
640  testChangesText('.', version)
641
642
643def testDemo(run_java, isSrc, version, jdk):
644  if os.path.exists('index'):
645    shutil.rmtree('index') # nuke any index from any previous iteration
646
647  print('    test demo with %s...' % jdk)
648  sep = ';' if cygwin else ':'
649  if isSrc:
650    # For source release, use the classpath for each module.
651    classPath = ['lucene/core/build/libs/lucene-core-%s.jar' % version,
652                 'lucene/demo/build/libs/lucene-demo-%s.jar' % version,
653                 'lucene/analysis/common/build/libs/lucene-analyzers-common-%s.jar' % version,
654                 'lucene/queryparser/build/libs/lucene-queryparser-%s.jar' % version]
655    cp = sep.join(classPath)
656    docsDir = 'lucene/core/src'
657    checkIndexCmd = 'java -ea -cp "%s" org.apache.lucene.index.CheckIndex index' % cp
658    indexFilesCmd = 'java -cp "%s" -Dsmoketester=true org.apache.lucene.demo.IndexFiles -index index -docs %s' % (cp, docsDir)
659    searchFilesCmd = 'java -cp "%s" org.apache.lucene.demo.SearchFiles -index index -query lucene' % cp
660  else:
661    # For binary release, set up module path.
662    cp = "--module-path %s" % (sep.join(["modules", "modules-thirdparty"]))
663    docsDir = 'docs'
664    checkIndexCmd = 'java -ea %s --module org.apache.lucene.core/org.apache.lucene.index.CheckIndex index' % cp
665    indexFilesCmd = 'java -Dsmoketester=true %s --module org.apache.lucene.demo/org.apache.lucene.demo.IndexFiles -index index -docs %s' % (cp, docsDir)
666    searchFilesCmd = 'java %s --module org.apache.lucene.demo/org.apache.lucene.demo.SearchFiles -index index -query lucene' % cp
667
668  run_java(indexFilesCmd, 'index.log')
669  run_java(searchFilesCmd, 'search.log')
670  reMatchingDocs = re.compile('(\d+) total matching documents')
671  m = reMatchingDocs.search(open('search.log', encoding='UTF-8').read())
672  if m is None:
673    raise RuntimeError('lucene demo\'s SearchFiles found no results')
674  else:
675    numHits = int(m.group(1))
676    if numHits < 100:
677      raise RuntimeError('lucene demo\'s SearchFiles found too few results: %s' % numHits)
678    print('      got %d hits for query "lucene"' % numHits)
679
680  print('    checkindex with %s...' % jdk)
681  run_java(checkIndexCmd, 'checkindex.log')
682  s = open('checkindex.log').read()
683  m = re.search(r'^\s+version=(.*?)$', s, re.MULTILINE)
684  if m is None:
685    raise RuntimeError('unable to locate version=NNN output from CheckIndex; see checkindex.log')
686  actualVersion = m.group(1)
687  if removeTrailingZeros(actualVersion) != removeTrailingZeros(version):
688    raise RuntimeError('wrong version from CheckIndex: got "%s" but expected "%s"' % (actualVersion, version))
689
690
691def removeTrailingZeros(version):
692  return re.sub(r'(\.0)*$', '', version)
693
694
695def checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
696  print('    download artifacts')
697  artifacts = []
698  artifactsURL = '%s/lucene/maven/org/apache/lucene/' % baseURL
699  targetDir = '%s/maven/org/apache/lucene' % tmpDir
700  if not os.path.exists(targetDir):
701    os.makedirs(targetDir)
702  crawl(artifacts, artifactsURL, targetDir)
703  print()
704  verifyPOMperBinaryArtifact(artifacts, version)
705  verifyMavenDigests(artifacts)
706  checkJavadocAndSourceArtifacts(artifacts, version)
707  verifyDeployedPOMsCoordinates(artifacts, version)
708  if isSigned:
709    verifyMavenSigs(tmpDir, artifacts, keysFile)
710
711  distFiles = getBinaryDistFiles(tmpDir, version, baseURL)
712  checkIdenticalMavenArtifacts(distFiles, artifacts, version)
713
714  checkAllJARs('%s/maven/org/apache/lucene' % tmpDir, gitRevision, version)
715
716
717def getBinaryDistFiles(tmpDir, version, baseURL):
718  distribution = 'lucene-%s.tgz' % version
719  if not os.path.exists('%s/%s' % (tmpDir, distribution)):
720    distURL = '%s/lucene/%s' % (baseURL, distribution)
721    print('    download %s...' % distribution, end=' ')
722    scriptutil.download(distribution, distURL, tmpDir, force_clean=FORCE_CLEAN)
723  destDir = '%s/unpack-lucene-getBinaryDistFiles' % tmpDir
724  if os.path.exists(destDir):
725    shutil.rmtree(destDir)
726  os.makedirs(destDir)
727  os.chdir(destDir)
728  print('    unpack %s...' % distribution)
729  unpackLogFile = '%s/unpack-%s-getBinaryDistFiles.log' % (tmpDir, distribution)
730  run('tar xzf %s/%s' % (tmpDir, distribution), unpackLogFile)
731  distributionFiles = []
732  for root, dirs, files in os.walk(destDir):
733    distributionFiles.extend([os.path.join(root, file) for file in files])
734  return distributionFiles
735
736
737def checkJavadocAndSourceArtifacts(artifacts, version):
738  print('    check for javadoc and sources artifacts...')
739  for artifact in artifacts:
740    if artifact.endswith(version + '.jar'):
741      javadocJar = artifact[:-4] + '-javadoc.jar'
742      if javadocJar not in artifacts:
743        raise RuntimeError('missing: %s' % javadocJar)
744      sourcesJar = artifact[:-4] + '-sources.jar'
745      if sourcesJar not in artifacts:
746        raise RuntimeError('missing: %s' % sourcesJar)
747
748
749def getZipFileEntries(fileName):
750  entries = []
751  with zipfile.ZipFile(fileName) as zf:
752    for zi in zf.infolist():
753      entries.append(zi.filename)
754  # Sort by name:
755  entries.sort()
756  return entries
757
758
759def checkIdenticalMavenArtifacts(distFiles, artifacts, version):
760  print('    verify that Maven artifacts are same as in the binary distribution...')
761  reJarWar = re.compile(r'%s\.[wj]ar$' % version)  # exclude *-javadoc.jar and *-sources.jar
762  distFilenames = dict()
763  for file in distFiles:
764    baseName = os.path.basename(file)
765    distFilenames[baseName] = file
766  for artifact in artifacts:
767    if reJarWar.search(artifact):
768      artifactFilename = os.path.basename(artifact)
769      if artifactFilename not in distFilenames:
770        raise RuntimeError('Maven artifact %s is not present in lucene binary distribution' % artifact)
771      else:
772        identical = filecmp.cmp(artifact, distFilenames[artifactFilename], shallow=False)
773        if not identical:
774          raise RuntimeError('Maven artifact %s is not identical to %s in lucene binary distribution'
775                % (artifact, distFilenames[artifactFilename]))
776
777
778def verifyMavenDigests(artifacts):
779  print("    verify Maven artifacts' md5/sha1 digests...")
780  reJarWarPom = re.compile(r'\.(?:[wj]ar|pom)$')
781  for artifactFile in [a for a in artifacts if reJarWarPom.search(a)]:
782    if artifactFile + '.md5' not in artifacts:
783      raise RuntimeError('missing: MD5 digest for %s' % artifactFile)
784    if artifactFile + '.sha1' not in artifacts:
785      raise RuntimeError('missing: SHA1 digest for %s' % artifactFile)
786    with open(artifactFile + '.md5', encoding='UTF-8') as md5File:
787      md5Expected = md5File.read().strip()
788    with open(artifactFile + '.sha1', encoding='UTF-8') as sha1File:
789      sha1Expected = sha1File.read().strip()
790    md5 = hashlib.md5()
791    sha1 = hashlib.sha1()
792    inputFile = open(artifactFile, 'rb')
793    while True:
794      bytes = inputFile.read(65536)
795      if len(bytes) == 0:
796        break
797      md5.update(bytes)
798      sha1.update(bytes)
799    inputFile.close()
800    md5Actual = md5.hexdigest()
801    sha1Actual = sha1.hexdigest()
802    if md5Actual != md5Expected:
803      raise RuntimeError('MD5 digest mismatch for %s: expected %s but got %s'
804                         % (artifactFile, md5Expected, md5Actual))
805    if sha1Actual != sha1Expected:
806      raise RuntimeError('SHA1 digest mismatch for %s: expected %s but got %s'
807                         % (artifactFile, sha1Expected, sha1Actual))
808
809
810def getPOMcoordinate(treeRoot):
811  namespace = '{http://maven.apache.org/POM/4.0.0}'
812  groupId = treeRoot.find('%sgroupId' % namespace)
813  if groupId is None:
814    groupId = treeRoot.find('{0}parent/{0}groupId'.format(namespace))
815  groupId = groupId.text.strip()
816  artifactId = treeRoot.find('%sartifactId' % namespace).text.strip()
817  version = treeRoot.find('%sversion' % namespace)
818  if version is None:
819    version = treeRoot.find('{0}parent/{0}version'.format(namespace))
820  version = version.text.strip()
821  packaging = treeRoot.find('%spackaging' % namespace)
822  packaging = 'jar' if packaging is None else packaging.text.strip()
823  return groupId, artifactId, packaging, version
824
825
826def verifyMavenSigs(tmpDir, artifacts, keysFile):
827  print('    verify maven artifact sigs', end=' ')
828
829  # Set up clean gpg world; import keys file:
830  gpgHomeDir = '%s/lucene.gpg' % tmpDir
831  if os.path.exists(gpgHomeDir):
832    shutil.rmtree(gpgHomeDir)
833  os.makedirs(gpgHomeDir, 0o700)
834  run('gpg --homedir %s --import %s' % (gpgHomeDir, keysFile),
835      '%s/lucene.gpg.import.log' % tmpDir)
836
837  reArtifacts = re.compile(r'\.(?:pom|[jw]ar)$')
838  for artifactFile in [a for a in artifacts if reArtifacts.search(a)]:
839    artifact = os.path.basename(artifactFile)
840    sigFile = '%s.asc' % artifactFile
841    # Test sig (this is done with a clean brand-new GPG world)
842    logFile = '%s/lucene.%s.gpg.verify.log' % (tmpDir, artifact)
843    run('gpg --display-charset utf-8 --homedir %s --verify %s %s' % (gpgHomeDir, sigFile, artifactFile),
844        logFile)
845
846    # Forward any GPG warnings, except the expected one (since it's a clean world)
847    print_warnings_in_file(logFile)
848
849    # Test trust (this is done with the real users config)
850    run('gpg --import %s' % keysFile,
851        '%s/lucene.gpg.trust.import.log' % tmpDir)
852    logFile = '%s/lucene.%s.gpg.trust.log' % (tmpDir, artifact)
853    run('gpg --display-charset utf-8 --verify %s %s' % (sigFile, artifactFile), logFile)
854    # Forward any GPG warnings:
855    print_warnings_in_file(logFile)
856
857    sys.stdout.write('.')
858  print()
859
860
861def print_warnings_in_file(file):
862  with open(file) as f:
863    for line in f.readlines():
864      if line.lower().find('warning') != -1 \
865          and line.find('WARNING: This key is not certified with a trusted signature') == -1 \
866              and line.find('WARNING: using insecure memory') == -1:
867        print('      GPG: %s' % line.strip())
868
869
870def verifyPOMperBinaryArtifact(artifacts, version):
871  print('    verify that each binary artifact has a deployed POM...')
872  reBinaryJarWar = re.compile(r'%s\.[jw]ar$' % re.escape(version))
873  for artifact in [a for a in artifacts if reBinaryJarWar.search(a)]:
874    POM = artifact[:-4] + '.pom'
875    if POM not in artifacts:
876      raise RuntimeError('missing: POM for %s' % artifact)
877
878
879def verifyDeployedPOMsCoordinates(artifacts, version):
880  """
881  verify that each POM's coordinate (drawn from its content) matches
882  its filepath, and verify that the corresponding artifact exists.
883  """
884  print("    verify deployed POMs' coordinates...")
885  for POM in [a for a in artifacts if a.endswith('.pom')]:
886    treeRoot = ET.parse(POM).getroot()
887    groupId, artifactId, packaging, POMversion = getPOMcoordinate(treeRoot)
888    POMpath = '%s/%s/%s/%s-%s.pom' \
889            % (groupId.replace('.', '/'), artifactId, version, artifactId, version)
890    if not POM.endswith(POMpath):
891      raise RuntimeError("Mismatch between POM coordinate %s:%s:%s and filepath: %s"
892                        % (groupId, artifactId, POMversion, POM))
893    # Verify that the corresponding artifact exists
894    artifact = POM[:-3] + packaging
895    if artifact not in artifacts:
896      raise RuntimeError('Missing corresponding .%s artifact for POM %s' % (packaging, POM))
897
898
899def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
900  for text, subURL in getDirEntries(urlString):
901    if text not in exclusions:
902      path = os.path.join(targetDir, text)
903      if text.endswith('/'):
904        if not os.path.exists(path):
905          os.makedirs(path)
906        crawl(downloadedFiles, subURL, path, exclusions)
907      else:
908        if not os.path.exists(path) or FORCE_CLEAN:
909          scriptutil.download(text, subURL, targetDir, quiet=True, force_clean=FORCE_CLEAN)
910        downloadedFiles.append(path)
911        sys.stdout.write('.')
912
913
914def make_java_config(parser, java18_home):
915  def _make_runner(java_home, version):
916    print('Java %s JAVA_HOME=%s' % (version, java_home))
917    if cygwin:
918      java_home = subprocess.check_output('cygpath -u "%s"' % java_home, shell=True).decode('utf-8').strip()
919    cmd_prefix = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % \
920                 (java_home, java_home, java_home)
921    s = subprocess.check_output('%s; java -version' % cmd_prefix,
922                                shell=True, stderr=subprocess.STDOUT).decode('utf-8')
923    if s.find(' version "%s' % version) == -1:
924      parser.error('got wrong version for java %s:\n%s' % (version, s))
925    def run_java(cmd, logfile):
926      run('%s; %s' % (cmd_prefix, cmd), logfile)
927    return run_java
928  java17_home =  os.environ.get('JAVA_HOME')
929  if java17_home is None:
930    parser.error('JAVA_HOME must be set')
931  run_java17 = _make_runner(java17_home, '17')
932  run_java18 = None
933  if java18_home is not None:
934    run_java18 = _make_runner(java18_home, '18')
935
936  jc = namedtuple('JavaConfig', 'run_java17 java17_home run_java18 java18_home')
937  return jc(run_java17, java17_home, run_java18, java18_home)
938
939version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
940revision_re = re.compile(r'rev-([a-f\d]+)')
941def parse_config():
942  epilogue = textwrap.dedent('''
943    Example usage:
944    python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/lucene/lucene-9.0.0-RC1-rev-c7510a0...
945  ''')
946  description = 'Utility to test a release.'
947  parser = argparse.ArgumentParser(description=description, epilog=epilogue,
948                                   formatter_class=argparse.RawDescriptionHelpFormatter)
949  parser.add_argument('--tmp-dir', metavar='PATH',
950                      help='Temporary directory to test inside, defaults to /tmp/smoke_lucene_$version_$revision')
951  parser.add_argument('--not-signed', dest='is_signed', action='store_false', default=True,
952                      help='Indicates the release is not signed')
953  parser.add_argument('--local-keys', metavar='PATH',
954                      help='Uses local KEYS file instead of fetching from https://archive.apache.org/dist/lucene/KEYS')
955  parser.add_argument('--revision',
956                      help='GIT revision number that release was built with, defaults to that in URL')
957  parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
958                      help='Version of the release, defaults to that in URL')
959  parser.add_argument('--test-java18', metavar='java18_home',
960                      help='Path to Java home directory, to run tests with if specified')
961  parser.add_argument('--download-only', action='store_true', default=False,
962                      help='Only perform download and sha hash check steps')
963  parser.add_argument('url', help='Url pointing to release to test')
964  parser.add_argument('test_args', nargs=argparse.REMAINDER,
965                      help='Arguments to pass to gradle for testing, e.g. -Dwhat=ever.')
966  c = parser.parse_args()
967
968  if c.version is not None:
969    if not version_re.match(c.version):
970      parser.error('version "%s" does not match format X.Y.Z[-ALPHA|-BETA]' % c.version)
971  else:
972    version_match = version_re.search(c.url)
973    if version_match is None:
974      parser.error('Could not find version in URL')
975    c.version = version_match.group(1)
976
977  if c.revision is None:
978    revision_match = revision_re.search(c.url)
979    if revision_match is None:
980      parser.error('Could not find revision in URL')
981    c.revision = revision_match.group(1)
982    print('Revision: %s' % c.revision)
983
984  if c.local_keys is not None and not os.path.exists(c.local_keys):
985    parser.error('Local KEYS file "%s" not found' % c.local_keys)
986
987  c.java = make_java_config(parser, c.test_java18)
988
989  if c.tmp_dir:
990    c.tmp_dir = os.path.abspath(c.tmp_dir)
991  else:
992    tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision)
993    c.tmp_dir = tmp
994    i = 1
995    while os.path.exists(c.tmp_dir):
996      c.tmp_dir = tmp + '_%d' % i
997      i += 1
998
999  return c
1000
1001reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE)
1002reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.', re.IGNORECASE)
1003
1004def getAllLuceneReleases():
1005  s = load('https://archive.apache.org/dist/lucene/java')
1006
1007  releases = set()
1008  for r in reVersion1, reVersion2:
1009    for tup in r.findall(s):
1010      if tup[-1].lower() == '-alpha':
1011        tup = tup[:3] + ('0',)
1012      elif tup[-1].lower() == '-beta':
1013        tup = tup[:3] + ('1',)
1014      elif tup[-1] == '':
1015        tup = tup[:3]
1016      else:
1017        raise RuntimeError('failed to parse version: %s' % tup[-1])
1018      releases.add(tuple(int(x) for x in tup))
1019
1020  l = list(releases)
1021  l.sort()
1022  return l
1023
1024
1025def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
1026
1027  print('    find all past Lucene releases...')
1028  allReleases = getAllLuceneReleases()
1029  #for tup in allReleases:
1030  #  print('  %s' % '.'.join(str(x) for x in tup))
1031
1032  testedIndicesPaths = glob.glob('%s/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/*-cfs.zip' % unpackPath)
1033  testedIndices = set()
1034
1035  reIndexName = re.compile(r'^[^.]*.(.*?)-cfs.zip')
1036  for name in testedIndicesPaths:
1037    basename = os.path.basename(name)
1038    version = reIndexName.fullmatch(basename).group(1)
1039    tup = tuple(version.split('.'))
1040    if len(tup) == 3:
1041      # ok
1042      tup = tuple(int(x) for x in tup)
1043    elif tup == ('4', '0', '0', '1'):
1044      # CONFUSING: this is the 4.0.0-alpha index??
1045      tup = 4, 0, 0, 0
1046    elif tup == ('4', '0', '0', '2'):
1047      # CONFUSING: this is the 4.0.0-beta index??
1048      tup = 4, 0, 0, 1
1049    elif basename == 'unsupported.5x-with-4x-segments-cfs.zip':
1050      # Mixed version test case; ignore it for our purposes because we only
1051      # tally up the "tests single Lucene version" indices
1052      continue
1053    elif basename == 'unsupported.5.0.0.singlesegment-cfs.zip':
1054      tup = 5, 0, 0
1055    else:
1056      raise RuntimeError('could not parse version %s' % name)
1057
1058    testedIndices.add(tup)
1059
1060  l = list(testedIndices)
1061  l.sort()
1062  if False:
1063    for release in l:
1064      print('  %s' % '.'.join(str(x) for x in release))
1065
1066  allReleases = set(allReleases)
1067
1068  for x in testedIndices:
1069    if x not in allReleases:
1070      # Curious: we test 1.9.0 index but it's not in the releases (I think it was pulled because of nasty bug?)
1071      if x != (1, 9, 0):
1072        raise RuntimeError('tested version=%s but it was not released?' % '.'.join(str(y) for y in x))
1073
1074  notTested = []
1075  for x in allReleases:
1076    if x not in testedIndices:
1077      releaseVersion = '.'.join(str(y) for y in x)
1078      if releaseVersion in ('1.4.3', '1.9.1', '2.3.1', '2.3.2'):
1079        # Exempt the dark ages indices
1080        continue
1081      if x >= tuple(int(y) for y in smokeVersion.split('.')):
1082        # Exempt versions not less than the one being smoke tested
1083        print('      Backcompat testing not required for release %s because it\'s not less than %s'
1084              % (releaseVersion, smokeVersion))
1085        continue
1086      notTested.append(x)
1087
1088  if len(notTested) > 0:
1089    notTested.sort()
1090    print('Releases that don\'t seem to be tested:')
1091    failed = True
1092    for x in notTested:
1093      print('  %s' % '.'.join(str(y) for y in x))
1094    raise RuntimeError('some releases are not tested by TestBackwardsCompatibility?')
1095  else:
1096    print('    success!')
1097
1098
1099def main():
1100  c = parse_config()
1101
1102  # Pick <major>.<minor> part of version and require script to be from same branch
1103  scriptVersion = re.search(r'((\d+).(\d+)).(\d+)', scriptutil.find_current_version()).group(1).strip()
1104  if not c.version.startswith(scriptVersion + '.'):
1105    raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
1106
1107  print('NOTE: output encoding is %s' % sys.stdout.encoding)
1108  smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, ' '.join(c.test_args),
1109            downloadOnly=c.download_only)
1110
1111
1112def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, testArgs, downloadOnly=False):
1113  startTime = datetime.datetime.now()
1114
1115  # Tests annotated @Nightly are more resource-intensive but often cover
1116  # important code paths. They're disabled by default to preserve a good
1117  # developer experience, but we enable them for smoke tests where we want good
1118  # coverage.
1119  testArgs = '-Dtests.nightly=true %s' % testArgs
1120
1121  # We also enable GUI tests in smoke tests (LUCENE-10531)
1122  testArgs = '-Dtests.gui=true %s' % testArgs
1123
1124  if FORCE_CLEAN:
1125    if os.path.exists(tmpDir):
1126      raise RuntimeError('temp dir %s exists; please remove first' % tmpDir)
1127
1128  if not os.path.exists(tmpDir):
1129    os.makedirs(tmpDir)
1130
1131  lucenePath = None
1132  print()
1133  print('Load release URL "%s"...' % baseURL)
1134  newBaseURL = unshortenURL(baseURL)
1135  if newBaseURL != baseURL:
1136    print('  unshortened: %s' % newBaseURL)
1137    baseURL = newBaseURL
1138
1139  for text, subURL in getDirEntries(baseURL):
1140    if text.lower().find('lucene') != -1:
1141      lucenePath = subURL
1142
1143  if lucenePath is None:
1144    raise RuntimeError('could not find lucene subdir')
1145
1146  print()
1147  print('Get KEYS...')
1148  if local_keys is not None:
1149    print("    Using local KEYS file %s" % local_keys)
1150    keysFile = local_keys
1151  else:
1152    keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
1153    print("    Downloading online KEYS file %s" % keysFileURL)
1154    scriptutil.download('KEYS', keysFileURL, tmpDir, force_clean=FORCE_CLEAN)
1155    keysFile = '%s/KEYS' % (tmpDir)
1156
1157  print()
1158  print('Test Lucene...')
1159  checkSigs(lucenePath, version, tmpDir, isSigned, keysFile)
1160  if not downloadOnly:
1161    unpackAndVerify(java, tmpDir, 'lucene-%s.tgz' % version, gitRevision, version, testArgs)
1162    unpackAndVerify(java, tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs)
1163    print()
1164    print('Test Maven artifacts...')
1165    checkMaven(baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
1166  else:
1167    print("\nLucene test done (--download-only specified)")
1168
1169  print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))
1170
1171
1172if __name__ == '__main__':
1173  try:
1174    main()
1175  except KeyboardInterrupt:
1176    print('Keyboard interrupt...exiting')
1177
1178