1d219b4ceSAdam Hornacek /* 2d219b4ceSAdam Hornacek * CDDL HEADER START 3d219b4ceSAdam Hornacek * 4d219b4ceSAdam Hornacek * The contents of this file are subject to the terms of the 5d219b4ceSAdam Hornacek * Common Development and Distribution License (the "License"). 6d219b4ceSAdam Hornacek * You may not use this file except in compliance with the License. 7d219b4ceSAdam Hornacek * 8d219b4ceSAdam Hornacek * See LICENSE.txt included in this distribution for the specific 9d219b4ceSAdam Hornacek * language governing permissions and limitations under the License. 10d219b4ceSAdam Hornacek * 11d219b4ceSAdam Hornacek * When distributing Covered Code, include this CDDL HEADER in each 12d219b4ceSAdam Hornacek * file and include the License file at LICENSE.txt. 13d219b4ceSAdam Hornacek * If applicable, add the following below this CDDL HEADER, with the 14d219b4ceSAdam Hornacek * fields enclosed by brackets "[]" replaced with your own identifying 15d219b4ceSAdam Hornacek * information: Portions Copyright [yyyy] [name of copyright owner] 16d219b4ceSAdam Hornacek * 17d219b4ceSAdam Hornacek * CDDL HEADER END 18d219b4ceSAdam Hornacek */ 19d219b4ceSAdam Hornacek 20d219b4ceSAdam Hornacek /* 21d219b4ceSAdam Hornacek * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. 22d219b4ceSAdam Hornacek * Portions Copyright (c) 2017-2018, Chris Fraire <cfraire@me.com>. 23d219b4ceSAdam Hornacek */ 24d219b4ceSAdam Hornacek 25d219b4ceSAdam Hornacek package org.opengrok.indexer.analysis.plain; 26d219b4ceSAdam Hornacek 27d219b4ceSAdam Hornacek import java.util.Locale; 28d219b4ceSAdam Hornacek import org.opengrok.indexer.analysis.JFlexSymbolMatcher; 29d219b4ceSAdam Hornacek %% 30d219b4ceSAdam Hornacek %public 31d219b4ceSAdam Hornacek %class PlainFullTokenizer 32d219b4ceSAdam Hornacek %extends JFlexSymbolMatcher 33d219b4ceSAdam Hornacek %unicode 34d219b4ceSAdam Hornacek %buffer 32766 35d219b4ceSAdam Hornacek %int 36d219b4ceSAdam Hornacek %include ../CommonLexer.lexh 37d219b4ceSAdam Hornacek %caseless 38d219b4ceSAdam Hornacek %char 39d219b4ceSAdam Hornacek 40d219b4ceSAdam Hornacek //WhiteSpace = [ \t\f\r]+|\n 41d219b4ceSAdam Hornacek Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]* 42d219b4ceSAdam Hornacek Number = [0-9]+|[0-9]+\.[0-9]+| "0[xX]" [0-9a-fA-F]+ 43d219b4ceSAdam Hornacek // No letters in the following, so no toLowerCase(Locale.ROOT) needed. 44*731916a8SAdam Hornacek Printable = [\@\$\%\^\&\-+=\?\.\:!\[\]\{\}*\/\|#\<\>\(\),;~] 45d219b4ceSAdam Hornacek 46d219b4ceSAdam Hornacek %% 47d219b4ceSAdam Hornacek {Identifier}|{Number}|{Printable} { 48d219b4ceSAdam Hornacek onSymbolMatched(yytext().toLowerCase(Locale.ROOT), yychar); 49d219b4ceSAdam Hornacek return yystate(); 50d219b4ceSAdam Hornacek } 51d219b4ceSAdam Hornacek [^] {} 52