1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Licensed to the Apache Software Foundation (ASF) under one or more 4# contributor license agreements. See the NOTICE file distributed with 5# this work for additional information regarding copyright ownership. 6# The ASF licenses this file to You under the Apache License, Version 2.0 7# (the "License"); you may not use this file except in compliance with 8# the License. You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18 19# For usage information, see: 20# 21# http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes 22 23 24import os 25import sys 26sys.path.append(os.path.dirname(__file__)) 27import scriptutil 28 29import argparse 30import urllib.error 31import urllib.request 32import re 33import shutil 34 35def create_and_add_index(source, indextype, index_version, current_version, temp_dir): 36 if not current_version.is_back_compat_with(index_version): 37 prefix = 'unsupported' 38 else: 39 prefix = { 40 'cfs': 'index', 41 'nocfs': 'index', 42 'sorted': 'sorted', 43 'moreterms': 'moreterms', 44 'dvupdates': 'dvupdates', 45 'emptyIndex': 'empty' 46 }[indextype] 47 if indextype in ('cfs', 'nocfs'): 48 dirname = 'index.%s' % indextype 49 filename = '%s.%s-%s.zip' % (prefix, index_version, indextype) 50 else: 51 dirname = indextype 52 filename = '%s.%s.zip' % (prefix, index_version) 53 54 print(' creating %s...' % filename, end='', flush=True) 55 module = 'backward-codecs' 56 index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/backward_index') 57 test_file = os.path.join(index_dir, filename) 58 if os.path.exists(os.path.join(index_dir, filename)): 59 print('uptodate') 60 return 61 62 test = { 63 'cfs': 'testCreateCFS', 64 'nocfs': 'testCreateNoCFS', 65 'sorted': 'testCreateSortedIndex', 66 'moreterms': 'testCreateMoreTermsIndex', 67 'dvupdates': 'testCreateIndexWithDocValuesUpdates', 68 'emptyIndex': 'testCreateEmptyIndex' 69 }[indextype] 70 gradle_args = ' '.join([ 71 '-Ptests.useSecurityManager=false', 72 '-p lucene/%s' % module, 73 'test', 74 '--tests TestBackwardsCompatibility.%s' % test, 75 '-Dtests.bwcdir=%s' % temp_dir, 76 '-Dtests.codec=default' 77 ]) 78 base_dir = os.getcwd() 79 bc_index_dir = os.path.join(temp_dir, dirname) 80 bc_index_file = os.path.join(bc_index_dir, filename) 81 82 if os.path.exists(bc_index_file): 83 print('alreadyexists') 84 else: 85 if os.path.exists(bc_index_dir): 86 shutil.rmtree(bc_index_dir) 87 os.chdir(source) 88 scriptutil.run('./gradlew %s' % gradle_args) 89 os.chdir(bc_index_dir) 90 scriptutil.run('zip %s *' % filename) 91 print('done') 92 93 print(' adding %s...' % filename, end='', flush=True) 94 scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir))) 95 os.chdir(base_dir) 96 scriptutil.run('rm -rf %s' % bc_index_dir) 97 print('done') 98 99def update_backcompat_tests(types, index_version, current_version): 100 print(' adding new indexes %s to backcompat tests...' % types, end='', flush=True) 101 module = 'lucene/backward-codecs' 102 filename = '%s/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java' % module 103 if not current_version.is_back_compat_with(index_version): 104 matcher = re.compile(r'final String\[\] unsupportedNames = {|};') 105 elif 'sorted' in types: 106 matcher = re.compile(r'static final String\[\] oldSortedNames = {|};') 107 else: 108 matcher = re.compile(r'static final String\[\] oldNames = {|};') 109 110 strip_dash_suffix_re = re.compile(r'-.*') 111 112 def find_version(x): 113 x = x.strip() 114 x = re.sub(strip_dash_suffix_re, '', x) # remove the -suffix if any 115 return scriptutil.Version.parse(x) 116 117 class Edit(object): 118 start = None 119 def __call__(self, buffer, match, line): 120 if self.start: 121 # find where this version should exist 122 i = len(buffer) - 1 123 previous_version_exists = not ('};' in line and buffer[-1].strip().endswith("{")) 124 if previous_version_exists: # Only look if there is a version here 125 v = find_version(buffer[i]) 126 while i >= self.start and v.on_or_after(index_version): 127 i -= 1 128 v = find_version(buffer[i]) 129 i += 1 # readjust since we skipped past by 1 130 131 # unfortunately python doesn't have a range remove from list... 132 # here we want to remove any previous references to the version we are adding 133 while i < len(buffer) and index_version.on_or_after(find_version(buffer[i])): 134 buffer.pop(i) 135 136 if i == len(buffer) and previous_version_exists and not buffer[-1].strip().endswith(","): 137 # add comma 138 buffer[-1] = buffer[-1].rstrip() + ",\n" 139 140 if previous_version_exists: 141 last = buffer[-1] 142 spaces = ' ' * (len(last) - len(last.lstrip())) 143 else: 144 spaces = ' ' 145 for (j, t) in enumerate(types): 146 if t == 'sorted': 147 newline = spaces + ('"sorted.%s"') % index_version 148 else: 149 newline = spaces + ('"%s-%s"' % (index_version, t)) 150 if j < len(types) - 1 or i < len(buffer): 151 newline += ',' 152 buffer.insert(i, newline + '\n') 153 i += 1 154 155 buffer.append(line) 156 return True 157 158 if 'Names = {' in line: 159 self.start = len(buffer) # location of first index name 160 buffer.append(line) 161 return False 162 163 changed = scriptutil.update_file(filename, matcher, Edit()) 164 print('done' if changed else 'uptodate') 165 166def check_backcompat_tests(): 167 print(' checking backcompat tests...', end='', flush=True) 168 scriptutil.run('./gradlew -p lucene/backward-codecs test --tests TestBackwardsCompatibility') 169 print('ok') 170 171def download_from_cdn(version, remotename, localname): 172 url = 'http://dlcdn.apache.org/lucene/java/%s/%s' % (version, remotename) 173 try: 174 urllib.request.urlretrieve(url, localname) 175 return True 176 except urllib.error.URLError as e: 177 if e.code == 404: 178 return False 179 raise e 180 181def download_from_archives(version, remotename, localname): 182 url = 'http://archive.apache.org/dist/lucene/java/%s/%s' % (version, remotename) 183 try: 184 urllib.request.urlretrieve(url, localname) 185 return True 186 except urllib.error.URLError as e: 187 if e.code == 404: 188 return False 189 raise e 190 191def download_release(version, temp_dir, force): 192 print(' downloading %s source release...' % version, end='', flush=True) 193 source = os.path.join(temp_dir, 'lucene-%s' % version) 194 if os.path.exists(source): 195 if force: 196 shutil.rmtree(source) 197 else: 198 print('uptodate') 199 return source 200 201 filename = 'lucene-%s-src.tgz' % version 202 source_tgz = os.path.join(temp_dir, filename) 203 if not download_from_cdn(version, filename, source_tgz) and \ 204 not download_from_archives(version, filename, source_tgz): 205 raise Exception('Could not find version %s in apache CDN or archives' % version) 206 207 olddir = os.getcwd() 208 os.chdir(temp_dir) 209 scriptutil.run('tar -xvzf %s' % source_tgz) 210 os.chdir(olddir) 211 print('done') 212 return source 213 214def read_config(): 215 parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, 216 description='''\ 217Add backcompat index and test for new version. See: 218http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes 219''') 220 parser.add_argument('--force', action='store_true', default=False, 221 help='Redownload the version and rebuild, even if it already exists') 222 parser.add_argument('--no-cleanup', dest='cleanup', action='store_false', default=True, 223 help='Do not cleanup the built indexes, so that they can be reused ' + 224 'for adding to another branch') 225 parser.add_argument('--temp-dir', metavar='DIR', default='/tmp/lucenebwc', 226 help='Temp directory to build backcompat indexes within') 227 parser.add_argument('version', type=scriptutil.Version.parse, 228 help='Version to add, of the form X.Y.Z') 229 c = parser.parse_args() 230 231 return c 232 233def main(): 234 c = read_config() 235 if not os.path.exists(c.temp_dir): 236 os.makedirs(c.temp_dir) 237 238 print('\nCreating backwards compatibility indexes') 239 source = download_release(c.version, c.temp_dir, c.force) 240 current_version = scriptutil.Version.parse(scriptutil.find_current_version()) 241 create_and_add_index(source, 'cfs', c.version, current_version, c.temp_dir) 242 create_and_add_index(source, 'nocfs', c.version, current_version, c.temp_dir) 243 should_make_sorted = current_version.is_back_compat_with(c.version) \ 244 and (c.version.major > 6 or (c.version.major == 6 and c.version.minor >= 2)) 245 if should_make_sorted: 246 create_and_add_index(source, 'sorted', c.version, current_version, c.temp_dir) 247 if c.version.minor == 0 and c.version.bugfix == 0 and current_version.is_back_compat_with(c.version): 248 create_and_add_index(source, 'moreterms', c.version, current_version, c.temp_dir) 249 create_and_add_index(source, 'dvupdates', c.version, current_version, c.temp_dir) 250 create_and_add_index(source, 'emptyIndex', c.version, current_version, c.temp_dir) 251 print ('\nMANUAL UPDATE REQUIRED: edit TestBackwardsCompatibility to enable moreterms, dvupdates, and empty index testing') 252 253 print('\nAdding backwards compatibility tests') 254 update_backcompat_tests(['cfs', 'nocfs'], c.version, current_version) 255 if should_make_sorted: 256 update_backcompat_tests(['sorted'], c.version, current_version) 257 258 print('\nTesting changes') 259 check_backcompat_tests() 260 261 if c.cleanup: 262 print('\nCleaning up') 263 print(' deleting %s...' % c.temp_dir, end='', flush=True) 264 shutil.rmtree(c.temp_dir) 265 print('done') 266 267 print() 268 269if __name__ == '__main__': 270 try: 271 main() 272 except KeyboardInterrupt: 273 print('\nRecieved Ctrl-C, exiting early') 274