Allow quoted identifiers like to be used like \"violin1", not just defined.

lily/lexer.ll

 %{ // -*- mode: c++; c-file-style: "linux"; indent-tabs-mode: t -*-
 /*
   This file is part of LilyPond, the GNU music typesetter.
 
   Copyright (C) 1996--2012 Han-Wen Nienhuys <hanwen@xs4all.nl>
                  Jan Nieuwenhuizen <janneke@gnu.org>
 
   LilyPond is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
@@ skipping 74 matching lines @@
 SCM lookup_markup_list_command (string s);
 bool is_valid_version (string s);
 
 
 #define start_quote() do {                      \
                 yy_push_state (quote);          \
                 yylval = SCM_EOL;               \
         } while (0)
 
 /*
-  The inside of \"violin1" is marked by nesting two quote modes as we

[Keith, 2012/10/28 17:38:11]

A start-condition named "command_quote" would be less mysterious than having
special meaning for "quote" within "quote".

That is, start-conditions <quote,command_quote> for scanning the innards of
quotes, then
<command_quote> \" {/* pop one state and return the command */}

-  don't have a dedicated command mode.
+  The inside of \"violin1" is marked by commandquote mode
 */
 
 #define start_command_quote() do {              \
-                yy_push_state (quote);          \
-                yy_push_state (quote);          \
+                yy_push_state (commandquote);   \
                 yylval = SCM_EOL;               \
         } while (0)
 
 #define yylval (*lexval_)
 
 #define yylloc (*lexloc_)
 
 #define YY_USER_ACTION  add_lexed_char (YYLeng ());
 
 
@@ skipping 12 matching lines @@
 %option stack
 %option never-interactive·
 %option warn
 
 %x extratoken
 %x chords
 %x figures
 %x incl
 %x lyrics
 %x longcomment
+%x maininput
 %x markup
 %x notes
 %x quote
+%x commandquote
 %x sourcefileline
 %x sourcefilename
 %x version
 
 /* The strategy concerning multibyte characters is to accept them but
  * call YYText_utf8 for patterns that might contain them, in order to
  * get a single code path responsible for flagging non-UTF-8 input:
  * Patterns for accepting only valid UTF-8 without backing up are
  * really hard to do and complex, and if nice error messages are
  * wanted, one would need patterns catching the invalid input as well.
  *
  * Since editors and operating environments don't necessarily behave
  * reasonably in the presence of mixed encodings, we flag encoding
  * errors also in identifiers, comments, and strings where it would be
  * conceivable to just transparently work with the byte string.  But
  * the whole point of caring about UTF-8 in here at all is too avoid
  * stranger errors later when input passes into backends or log files
  * or console output or error messages.
  */
 
 A               [a-zA-Z\200-\377]
 AA              {A}|_
 N               [0-9]
 ANY_CHAR        (.|\n)
 WORD            {A}([-_]{A}|{A})*
 COMMAND         \\{WORD}
-
+/* SPECIAL category is for every letter that needs to get passed to
+ * the parser rather than being redefinable by the user */
+SPECIAL         [-+*/=<>{}!?_^'',.:]
+SHORTHAND       (.|\\.)
 UNSIGNED        {N}+
 E_UNSIGNED      \\{N}+
 FRACTION        {N}+\/{N}+
 INT             -?{UNSIGNED}
 REAL            ({INT}\.{N}*)|(-?\.{N}+)
+STRICTREAL      {UNSIGNED}\.{UNSIGNED}
 WHITE           [ \n\t\f\r]
 HORIZONTALWHITE         [ \t]
 BLACK           [^ \n\t\f\r]
 RESTNAME        [rs]
 ESCAPED         [nt\\''""]
 EXTENDER        __
 HYPHEN          --
 BOM_UTF8        \357\273\277
 
 %%
@@ skipping 41 matching lines @@
       LexerWarning (_ ("stray UTF-8 BOM encountered").c_str ());
       // exit (1);
     }
   debug_output (_ ("Skipping UTF-8 BOM"));
 }
 
 <INITIAL,chords,figures,incl,lyrics,markup,notes>{
   "%{"  {
         yy_push_state (longcomment);
   }
-  %[^{\n\r][^\n\r]*[\n\r]       {
+  %[^{\n\r][^\n\r]*[\n\r]?      {
           (void) YYText_utf8 ();
   }
-  %[^{\n\r]     { // backup rule
-          (void) YYText_utf8 ();
-  }
-  %[\n\r]       {
-  }
-  %[^{\n\r][^\n\r]*     {
-          (void) YYText_utf8 ();
+  %[\n\r]?      {
   }
   {WHITE}+      {
 
   }
 }
 
 <INITIAL,notes,figures,chords,markup>{
         \"              {
                 start_quote ();
         }
@@ skipping 66 matching lines @@
         "%"+"}"         {
                 yy_pop_state ();
         }
 }
 
 
 <INITIAL,chords,lyrics,notes,figures>\\maininput           {
         if (!is_main_input_)
         {
                 start_main_input ();
+                main_input_level_ = include_stack_.size ();
                 is_main_input_ = true;
+                int state = YYSTATE;
+                yy_push_state (maininput);
+                yy_push_state (state);
         }
         else
-                error (_ ("\\maininput not allowed outside init files"));
+                LexerError (_ ("\\maininput not allowed outside init files").c_str ());
 }
 
 <INITIAL,chords,lyrics,figures,notes>\\include           {
         yy_push_state (incl);
 }
 <incl>\"[^""]*\"   { /* got the include file name */
         string s (YYText_utf8 ()+1);
         s = s.substr (0, s.rfind ('"'));
 
         new_input (s, sources_);
@@ skipping 39 matching lines @@
                 LexerError (_ ("string expected after \\include").c_str ());
                 if (sval != SCM_UNDEFINED) {
                         SCM err = scm_current_error_port ();
                         scm_puts ("This value was found instead: ", err);
                         scm_display (sval, err);
                 }
         }
 }
 
 <incl,version,sourcefilename>\"[^""]*   { // backup rule
-        error (_ ("end quote missing"));
-        exit (1);
+        LexerError (_ ("end quote missing").c_str ());
+        yy_pop_state ();
 }
 
     /* Flex picks the longest matching pattern including trailing
      * contexts.  Without the backup pattern, r-. does not trigger the
      * {RESTNAME} rule but rather the {WORD}/[-_] rule coming later,
      * needed for avoiding backup states.
      */
 
 <chords,notes,figures>{RESTNAME}/[-_]   |  // pseudo backup rule
 <chords,notes,figures>{RESTNAME}        {
@@ skipping 80 matching lines @@
                 return FIGURE_SPACE;
         }
         \>              {
                 yylval = SCM_UNSPECIFIED;
                 return FIGURE_CLOSE;
         }
         \<      {
                 yylval = SCM_UNSPECIFIED;
                 return FIGURE_OPEN;
         }
+        \\\+    {
+                yylval = SCM_UNSPECIFIED;
+                return E_PLUS;
+        }
+        \\!     {
+                yylval = SCM_UNSPECIFIED;
+                return E_EXCLAMATION;
+        }
+        \\\\    {
+                yylval = SCM_UNSPECIFIED;
+                return E_BACKSLASH;
+        }
+        [][]    {
+                yylval = SCM_UNSPECIFIED;
+                return  YYText ()[0];
+        }
 }
 
 <notes,figures>{
         {WORD}/[-_]     | // backup rule
         {WORD}  {
                 return scan_bare_word (YYText_utf8 ());
         }
         \\\"    {
                 start_command_quote ();
         }
         {COMMAND}/[-_]  | // backup rule
         {COMMAND}       {
                 return scan_escaped_word (YYText_utf8 () + 1);·
         }
         {FRACTION}      {
                 yylval =  scan_fraction (YYText ());
                 return FRACTION;
         }
-        {UNSIGNED}/\/   | // backup rule
-        {UNSIGNED}              {
+        {STRICTREAL}    {
+                yylval = scm_c_read_string (YYText ());
+                return REAL;
+        }
+        {UNSIGNED}/[/.] | // backup rule
+        {UNSIGNED}      {
                 yylval = scm_c_read_string (YYText ());
                 return UNSIGNED;
         }
         {E_UNSIGNED}    {
                 yylval = scm_c_read_string (YYText () + 1);
                 return E_UNSIGNED;
         }
 }
 
-<quote>{
+<quote,commandquote>{
         \\{ESCAPED}     {
                 char c = escaped_char (YYText ()[1]);
                 yylval = scm_cons (scm_from_locale_stringn (&c, 1),
                                    yylval);
         }
         [^\\""]+        {
                 yylval = scm_cons (scm_from_locale_string (YYText_utf8 ()),
                                    yylval);
         }
         \"      {
 
-                yy_pop_state ();
-
                 /* yylval is union. Must remember STRING before setting SCM*/
 
                 yylval = scm_string_concatenate_reverse (yylval,
                                                          SCM_UNDEFINED,
                                                          SCM_UNDEFINED);
 
-                if (get_state () == quote) {
+                if (get_state () == commandquote) {
                         yy_pop_state ();
                         return scan_escaped_word (ly_scm2string (yylval));
                 }
 
-                return is_lyric_state () ? LYRICS_STRING : STRING;
+                yy_pop_state ();
+
+                return STRING;
         }
         \\      {
                 yylval = scm_cons (scm_from_locale_string (YYText ()),
                                    yylval);
         }
 }
 
 <lyrics>{
         \" {
                 start_quote ();
         }
         {FRACTION}      {
                 yylval =  scan_fraction (YYText ());
                 return FRACTION;
         }
-        {UNSIGNED}/\/   | // backup rule
+        {STRICTREAL}    {
+                yylval = scm_c_read_string (YYText ());
+                return REAL;
+        }
+        {UNSIGNED}/[/.] | // backup rule
         {UNSIGNED}              {
                 yylval = scm_c_read_string (YYText ());
                 return UNSIGNED;
         }
         \\\"    {
                 start_command_quote ();
         }
         {COMMAND}/[-_]  | // backup rule
         {COMMAND}       {
                 return scan_escaped_word (YYText_utf8 () + 1);
         }
-        /* Characters needed to express durations, assignments, barchecks */
-        [*.=|]  {
+        \\.|\|  {
+                // UTF-8 already covered by COMMAND
+                return scan_shorthand (YYText ());
+        }
+        /* Characters needed to express durations, assignments */
+        [*.=]   {
                 yylval = SCM_UNSPECIFIED;
                 return YYText ()[0];
         }
         [^$#{}\"\\ \t\n\r\f0-9]+ {
                 /* ugr. This sux. */
                 string s (YYText_utf8 ());
                 yylval = SCM_UNSPECIFIED;
                 if (s == "__")
                         return EXTENDER;
                 if (s == "--")
                         return HYPHEN;
                 s = lyric_fudge (s);
                 yylval = ly_string2scm (s);
 
-                return LYRICS_STRING;
+                return STRING;
         }
         /* This should really just cover {} */
-        . {
-                yylval = SCM_UNSPECIFIED;
-                return YYText ()[0]; // above catches all multibytes.
+        [{}] {
+                yylval = SCM_UNSPECIFIED;
+                return YYText ()[0];
         }
 }
 <chords>{
         {WORD}/[-_]     | // backup rule
         {WORD}  {
                 return scan_bare_word (YYText_utf8 ());
         }
         \\\"    {
                 start_command_quote ();
         }
@@ skipping 22 matching lines @@
                 yylval = SCM_UNSPECIFIED;
                 return CHORD_BASS;
         }
         \/  {
                 yylval = SCM_UNSPECIFIED;
                 return CHORD_SLASH;
         }
         \^  {
                 yylval = SCM_UNSPECIFIED;
                 return CHORD_CARET;
-        }
-        . {
-                yylval = SCM_UNSPECIFIED;
-                return YYText ()[0]; // WORD catches all multibyte.
         }
 }
 
 
 <markup>{
         \\score {
                 yylval = SCM_UNSPECIFIED;
                 return SCORE;
         }
         \\\"    {
@@ skipping 49 matching lines @@
                     push_extra_token(EXPECT_SCM, predicate);
                 }
                 return token_type;
         }
         [^$#{}\"\\ \t\n\r\f]+ {
                 string s (YYText_utf8 ());·
 
                 yylval = ly_string2scm (s);
                 return STRING;
         }
-        .  {
-                yylval = SCM_UNSPECIFIED;
-                return YYText ()[0];  // Above is catchall for multibyte
+        [{}]  {
+                yylval = SCM_UNSPECIFIED;
+                return YYText ()[0];
         }
 }
 
 <longcomment><<EOF>> {
                 LexerError (_ ("EOF found inside a comment").c_str ());
-                is_main_input_ = false; // should be safe , can't have \include in --safe.
-                yylval = SCM_UNSPECIFIED;
-                if (!close_input ())
-                  yyterminate (); // can't move this, since it actually rets a YY_NULL
-        }
+                yy_pop_state ();
+        }
+
+<quote,commandquote><<EOF>> {
+        LexerError (_ ("EOF found inside string").c_str ());
+        yy_pop_state ();
+}
 
 <<EOF>> {
         yylval = SCM_UNSPECIFIED;
         if (is_main_input_)
         {
-                /* 2 = init.ly + current file.
-                   > because we're before closing, but is_main_input_ should
-                   reflect after.
-                */·
-                is_main_input_ = include_stack_.size () > 2;
+                is_main_input_ = include_stack_.size () > main_input_level_;
+                if (!is_main_input_)
+                {
+                        main_input_level_ = 0;
+                        pop_state ();
+                        if (YYSTATE != maininput)
+                        {
+                                LexerError (_ ("Unfinished main input").c_str ());
+                                do {
+                                        yy_pop_state ();
+                                } while (YYSTATE != maininput);
+                        }
+                        extra_tokens_ = SCM_EOL;
+                        yy_pop_state ();
+                }
                 if (!close_input () || !is_main_input_)
                 /* Returns YY_NULL */
                         yyterminate ();
         }
         else if (!close_input ())
                 /* Returns YY_NULL */
                 yyterminate ();
+}
+
+<maininput>{ANY_CHAR} {
+        while (include_stack_.size () > main_input_level_
+               && close_input ())
+                ;
+        yyterminate ();
 }
 
 <INITIAL>{
         {WORD}/[-_]     | // backup rule
         {WORD}  {
                 return scan_bare_word (YYText_utf8 ());
         }
         \\\"    {
                 start_command_quote ();
         }
@@ skipping 14 matching lines @@
         return REAL;
 }
 
 {UNSIGNED}/\/   | // backup rule
 {UNSIGNED}      {
         yylval = scm_c_read_string (YYText ());
         return UNSIGNED;
 }
 
 
-[{}]    {
+-/\.    { // backup rule
         yylval = SCM_UNSPECIFIED;
         return YYText ()[0];
 }
 
--/\.    | // backup rule
-[*:=]           {
+<INITIAL,chords,lyrics,figures,notes>{SPECIAL}  {
         yylval = SCM_UNSPECIFIED;
         return YYText ()[0];
 }
 
-<INITIAL,notes,figures>.        {
-        yylval = SCM_UNSPECIFIED;
-        return YYText ()[0];
-}
-
-<INITIAL,lyrics,notes,figures>\\. {
-    yylval = SCM_UNSPECIFIED;
-    char c = YYText ()[1];
-
-    switch (c) {
-    case '>':
-        return E_ANGLE_CLOSE;
-    case '<':
-        return E_ANGLE_OPEN;
-    case '!':
-        return E_EXCLAMATION;
-    case '(':
-        return E_OPEN;
-    case ')':
-        return E_CLOSE;
-    case '[':
-        return E_BRACKET_OPEN;
-    case '+':
-        return E_PLUS;
-    case ']':
-        return E_BRACKET_CLOSE;
-    case '~':
-        return E_TILDE;
-    case '\\':
-        return E_BACKSLASH;
-
-    default:
-        return E_CHAR;
-    }
+<INITIAL,chords,lyrics,figures,notes>{SHORTHAND}        {
+        return scan_shorthand (YYText_utf8 ()); // should not be utf-8
 }
 
 <*>.[\200-\277]*        {
         string msg = _f ("invalid character: `%s'", YYText_utf8 ());
         LexerError (msg.c_str ());
         yylval = SCM_UNSPECIFIED;
         return '%';  // Better not return half a utf8 character.
 }
 
 %%
@@ skipping 72 matching lines @@
 Lily_lexer::pop_state ()
 {
         bool extra = (YYSTATE == extratoken);
 
         if (extra)
                 yy_pop_state ();
 
         if (YYSTATE == notes || YYSTATE == chords)
                 pitchname_tab_stack_ = scm_cdr (pitchname_tab_stack_);
 
-        yy_pop_state ();
+        // don't cross the maininput threshold
+        if (YYSTATE != maininput)
+                yy_pop_state ();
 
         if (extra) {
                 hidden_state_ = YYSTATE;
                 yy_push_state (extratoken);
         }
 }
 
 int
 Lily_lexer::identifier_type (SCM sid)
 {
         int k = try_special_identifiers (&yylval , sid);
         return k >= 0  ? k : SCM_IDENTIFIER;
 }
 
 
 int
-Lily_lexer::scan_escaped_word (string str)
+Lily_lexer::scan_escaped_word (const string &str)
 {
         // use more SCM for this.
 
 //      SCM sym = ly_symbol2scm (str.c_str ());
 
         yylval = SCM_UNSPECIFIED;
         int i = lookup_keyword (str);
-        if (i == MARKUP && is_lyric_state ())
-                return LYRIC_MARKUP;
+
         if (i != -1)
                 return i;
 
         SCM sid = lookup_identifier (str);
+        if (Music *m = unsmob_music (sid))
+        {
+                m->set_spot (override_input (last_input_));
+        }
+
         if (sid != SCM_UNDEFINED)
                 return scan_scm_id (sid);
 
         string msg (_f ("unknown escaped string: `\\%s'", str));········
+        LexerError (msg.c_str ());
+
+        yylval = ly_string2scm (str);
+
+        return STRING;
+}
+
+int
+Lily_lexer::scan_shorthand (const string &str)
+{
+        SCM sid = lookup_identifier (str);
+        if (Music *m = unsmob_music (sid))
+        {
+                m->set_spot (override_input (last_input_));
+        }
+
+        if (sid != SCM_UNDEFINED)
+                return scan_scm_id (sid);
+
+        string msg (_f ("undefined character or shorthand: %s", str));··
         LexerError (msg.c_str ());
 
         yylval = ly_string2scm (str);
 
         return STRING;
 }
 
 int
 Lily_lexer::scan_scm_id (SCM sid)
 {
@@ skipping 45 matching lines @@
                         if (!scm_is_eq (optional, SCM_UNDEFINED))
                                 push_extra_token (EXPECT_OPTIONAL, optional);
                 }
                 return funtype;
         }
         yylval = sid;
         return identifier_type (sid);
 }
 
 int
-Lily_lexer::scan_bare_word (string str)
+Lily_lexer::scan_bare_word (const string &str)
 {
         SCM sym = ly_symbol2scm (str.c_str ());
         if ((YYSTATE == notes) || (YYSTATE == chords)) {
                 SCM handle = SCM_BOOL_F;
                 if (scm_is_pair (pitchname_tab_stack_))
                         handle = scm_hashq_get_handle (scm_cdar (pitchname_tab_stack_), sym);
 ················
                 if (scm_is_pair (handle)) {
                         yylval = scm_cdr (handle);
                         if (unsmob_pitch (yylval))
@@ skipping 69 matching lines @@
         {
                 error_level_ = 1;
                 return SCM_UNSPECIFIED;
         }
 
         if (extra_token && SCM_VALUESP (sval))
         {
                 sval = scm_struct_ref (sval, SCM_INUM0);
 
                 if (scm_is_pair (sval)) {
-                        for (SCM v = scm_reverse (scm_cdr (sval));
-                             scm_is_pair (v);
-                             v = scm_cdr (v))
+                        for (SCM p = scm_reverse (scm_cdr (sval));
+                             scm_is_pair (p);
+                             p = scm_cdr (p))
                         {
+                                SCM v = scm_car (p);
+                                if (Music *m = unsmob_music (v))
+                                {
+                                        if (!unsmob_input (m->get_property ("origin")))
+                                                m->set_spot (override_input (last_input_));
+                                }
+········································
                                 int token;
                                 switch (extra_token) {
                                 case '$':
-                                        token = scan_scm_id (scm_car (v));
+                                        token = scan_scm_id (v);
                                         if (!scm_is_eq (yylval, SCM_UNSPECIFIED))
                                                 push_extra_token (token, yylval);
                                         break;
                                 case '#':
-                                        push_extra_token (SCM_IDENTIFIER, scm_car (v));
+                                        push_extra_token (SCM_IDENTIFIER, v);
                                         break;
                                 }
                         }
                         sval = scm_car (sval);
                 } else
                         sval = SCM_UNSPECIFIED;
+        }
+
+        if (Music *m = unsmob_music (sval))
+        {
+                if (!unsmob_input (m->get_property ("origin")))
+                        m->set_spot (override_input (last_input_));
         }
 
         return sval;
 }
 
 /* Check for valid UTF-8 that has no overlong or surrogate codes and
    is in the range 0-0x10ffff */
 
 const char *
 Lily_lexer::YYText_utf8 ()
@@ skipping 221 matching lines @@
 {
    (void) yy_start_stack_ptr;
    (void) yy_start_stack_depth;
    (void) yy_start_stack;
    (void) yy_push_state;
    (void) yy_pop_state;
    (void) yy_top_state;
    (void) silence_lexer_warnings;
 }
 #endif