1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * See LICENSE.txt included in this distribution for the specific 9 * language governing permissions and limitations under the License. 10 * 11 * When distributing Covered Code, include this CDDL HEADER in each 12 * file and include the License file at LICENSE.txt. 13 * If applicable, add the following below this CDDL HEADER, with the 14 * fields enclosed by brackets "[]" replaced with your own identifying 15 * information: Portions Copyright [yyyy] [name of copyright owner] 16 * 17 * CDDL HEADER END 18 */ 19 20 /* 21 * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. 22 * Portions Copyright (c) 2017-2018, Chris Fraire <cfraire@me.com>. 23 */ 24 25 package org.opengrok.indexer.analysis.plain; 26 27 import java.util.Locale; 28 import org.opengrok.indexer.analysis.JFlexSymbolMatcher; 29 %% 30 %public 31 %class PlainFullTokenizer 32 %extends JFlexSymbolMatcher 33 %unicode 34 %buffer 32766 35 %int 36 %include ../CommonLexer.lexh 37 %caseless 38 %char 39 40 //WhiteSpace = [ \t\f\r]+|\n 41 Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]* 42 Number = [0-9]+|[0-9]+\.[0-9]+| "0[xX]" [0-9a-fA-F]+ 43 // No letters in the following, so no toLowerCase(Locale.ROOT) needed. 44 Printable = [\@\$\%\^\&\-+=\?\.\:!\[\]\{\}*\/\|#\<\>\(\),;~] 45 46 %% 47 {Identifier}|{Number}|{Printable} { 48 onSymbolMatched(yytext().toLowerCase(Locale.ROOT), yychar); 49 return yystate(); 50 } 51 [^] {} 52