1*d219b4ceSAdam Hornacek /* 2*d219b4ceSAdam Hornacek * CDDL HEADER START 3*d219b4ceSAdam Hornacek * 4*d219b4ceSAdam Hornacek * The contents of this file are subject to the terms of the 5*d219b4ceSAdam Hornacek * Common Development and Distribution License (the "License"). 6*d219b4ceSAdam Hornacek * You may not use this file except in compliance with the License. 7*d219b4ceSAdam Hornacek * 8*d219b4ceSAdam Hornacek * See LICENSE.txt included in this distribution for the specific 9*d219b4ceSAdam Hornacek * language governing permissions and limitations under the License. 10*d219b4ceSAdam Hornacek * 11*d219b4ceSAdam Hornacek * When distributing Covered Code, include this CDDL HEADER in each 12*d219b4ceSAdam Hornacek * file and include the License file at LICENSE.txt. 13*d219b4ceSAdam Hornacek * If applicable, add the following below this CDDL HEADER, with the 14*d219b4ceSAdam Hornacek * fields enclosed by brackets "[]" replaced with your own identifying 15*d219b4ceSAdam Hornacek * information: Portions Copyright [yyyy] [name of copyright owner] 16*d219b4ceSAdam Hornacek * 17*d219b4ceSAdam Hornacek * CDDL HEADER END 18*d219b4ceSAdam Hornacek */ 19*d219b4ceSAdam Hornacek 20*d219b4ceSAdam Hornacek /* 21*d219b4ceSAdam Hornacek * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. 22*d219b4ceSAdam Hornacek * Portions Copyright (c) 2017-2018, Chris Fraire <cfraire@me.com>. 23*d219b4ceSAdam Hornacek */ 24*d219b4ceSAdam Hornacek 25*d219b4ceSAdam Hornacek package org.opengrok.indexer.analysis.plain; 26*d219b4ceSAdam Hornacek 27*d219b4ceSAdam Hornacek import java.util.Locale; 28*d219b4ceSAdam Hornacek import org.opengrok.indexer.analysis.JFlexSymbolMatcher; 29*d219b4ceSAdam Hornacek %% 30*d219b4ceSAdam Hornacek %public 31*d219b4ceSAdam Hornacek %class PlainFullTokenizer 32*d219b4ceSAdam Hornacek %extends JFlexSymbolMatcher 33*d219b4ceSAdam Hornacek %unicode 34*d219b4ceSAdam Hornacek %buffer 32766 35*d219b4ceSAdam Hornacek %int 36*d219b4ceSAdam Hornacek %include ../CommonLexer.lexh 37*d219b4ceSAdam Hornacek %caseless 38*d219b4ceSAdam Hornacek %char 39*d219b4ceSAdam Hornacek 40*d219b4ceSAdam Hornacek //WhiteSpace = [ \t\f\r]+|\n 41*d219b4ceSAdam Hornacek Identifier = [a-zA-Z\p{Letter}_] [a-zA-Z\p{Letter}0-9\p{Number}_]* 42*d219b4ceSAdam Hornacek Number = [0-9]+|[0-9]+\.[0-9]+| "0[xX]" [0-9a-fA-F]+ 43*d219b4ceSAdam Hornacek // No letters in the following, so no toLowerCase(Locale.ROOT) needed. 44*d219b4ceSAdam Hornacek Printable = [\@\$\%\^\&\-+=\?\.\:] 45*d219b4ceSAdam Hornacek 46*d219b4ceSAdam Hornacek %% 47*d219b4ceSAdam Hornacek {Identifier}|{Number}|{Printable} { 48*d219b4ceSAdam Hornacek onSymbolMatched(yytext().toLowerCase(Locale.ROOT), yychar); 49*d219b4ceSAdam Hornacek return yystate(); 50*d219b4ceSAdam Hornacek } 51*d219b4ceSAdam Hornacek [^] {} 52