Allow preserving sigil metadata in container cursor to quoted

Closes #14243.

Author: josevalim

lib/elixir/lib/code/fragment.ex

@@ -1234,14 +1234,43 @@ defmodule Code.Fragment do
       the cursor. This is necessary to correctly complete anonymous functions
       and the left-hand side of `->`
 
+    * `:preserve_sigils` (since v1.20.0) - preserve sigil cursor location
+      (see "Tracking sigils" section below)
+
+  ## Tracking sigils
+
+  The `:preserve_sigils` option can be used to track cursor positions inside
+  a sigil.
+
+  If the sigil is terminated abruptly, the `sigil_*` call will have the cursor
+  as the second argument:
+
+      iex> Code.Fragment.container_cursor_to_quoted("~r/foo", preserve_sigils: true)
+      {:ok,
+      {:sigil_r, [delimiter: "/", line: 1],
+        [{:<<>>, [line: 1], ["foo"]}, {:__cursor__, [line: 1, column: 7], []}]}}
+
+  In case the sigil is completed and has zero or more modifiers, the cursor will
+  be nested in the list, with all previous delimiters specified:
+
+      iex> Code.Fragment.container_cursor_to_quoted("~r/foo/i", preserve_sigils: true)
+      {:ok,
+       {:sigil_r, [delimiter: "/", line: 1],
+        [{:<<>>, [line: 1], ["foo"]}, [105, {:__cursor__, [line: 1, column: 9], []}]]}}
+
+  If the cursor is after the sigil, then it is discarded as everything else:
+
+      iex> Code.Fragment.container_cursor_to_quoted("~r/foo/i ", preserve_sigils: true)
+      {:ok, {:__cursor__, [line: 1], []}}
   """
   @doc since: "1.13.0"
   @spec container_cursor_to_quoted(List.Chars.t(), container_cursor_to_quoted_opts()) ::
           {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}}
   def container_cursor_to_quoted(fragment, opts \\ []) do
     {trailing_fragment, opts} = Keyword.pop(opts, :trailing_fragment)
+    {preserve_sigils?, opts} = Keyword.pop(opts, :preserve_sigils, false)
     opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder])
-    opts = [check_terminators: {:cursor, []}] ++ opts
+    opts = [check_terminators: {:cursor, preserve_sigils?, []}] ++ opts
 
     file = Keyword.get(opts, :file, "nofile")
     line = Keyword.get(opts, :line, 1)
@@ -1272,7 +1301,7 @@ defmodule Code.Fragment do
                  Enum.split_while(rev_terminators, &(elem(&1, 0) not in [:do, :fn])),
                true <- maybe_missing_stab?(rev_tokens, true),
                opts =
-                 Keyword.put(opts, :check_terminators, {:cursor, before_start}),
+                 Keyword.put(opts, :check_terminators, {:cursor, false, before_start}),
                {:error, {meta, _, ~c"end"}, _rest, _warnings, trailing_rev_tokens} <-
                  :elixir_tokenizer.tokenize(to_charlist(trailing_fragment), line, column, opts) do
             trailing_tokens =

lib/elixir/src/elixir_interpolation.erl

@@ -20,10 +20,10 @@ extract([], _Buffer, _Output, Line, Column, #elixir_tokenizer{cursor_completion=
   {error, {string, Line, Column, io_lib:format("missing terminator: ~ts", [[Last]]), []}};
 
 extract([], Buffer, Output, Line, Column, Scope, _Interpol, _Last) ->
-  finish_extraction([], Buffer, Output, Line, Column, Scope);
+  finish_extraction([], false, Buffer, Output, Line, Column, Scope);
 
 extract([Last | Rest], Buffer, Output, Line, Column, Scope, _Interpol, Last) ->
-  finish_extraction(Rest, Buffer, Output, Line, Column + 1, Scope);
+  finish_extraction(Rest, true, Buffer, Output, Line, Column + 1, Scope);
 
 %% Going through the string
 
@@ -113,7 +113,7 @@ extract_char(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) ->
 extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, [H,H,H] = Last) ->
   case strip_horizontal_space(Rest, Buffer, 1) of
     {[H,H,H|NewRest], _NewBuffer, Column} ->
-      finish_extraction(NewRest, Buffer, Output, Line + 1, Column + 3, Scope);
+      finish_extraction(NewRest, true, Buffer, Output, Line + 1, Column + 3, Scope);
     {NewRest, NewBuffer, Column} ->
       extract(NewRest, NewBuffer, Output, Line + 1, Column, Scope, Interpol, Last)
   end;
@@ -267,13 +267,13 @@ unescape_map(E)  -> E.
 
 % Extract Helpers
 
-finish_extraction(Remaining, Buffer, Output, Line, Column, Scope) ->
+finish_extraction(Remaining, Done, Buffer, Output, Line, Column, Scope) ->
   Final = case build_string(Buffer, Output) of
     [] -> [[]];
     F  -> F
   end,
 
-  {Line, Column, lists:reverse(Final), Remaining, Scope}.
+  {Line, Column, lists:reverse(Final), Remaining, Done, Scope}.
 
 build_string([], Output) -> Output;
 build_string(Buffer, Output) -> [lists:reverse(Buffer) | Output].

lib/elixir/src/elixir_tokenizer.erl

@@ -121,8 +121,8 @@ tokenize(String, Line, Column, Opts) ->
     lists:foldl(fun
       ({check_terminators, false}, Acc) ->
         Acc#elixir_tokenizer{cursor_completion=false, terminators=none};
-      ({check_terminators, {cursor, Terminators}}, Acc) ->
-        Acc#elixir_tokenizer{cursor_completion=prune_and_cursor, terminators=Terminators};
+      ({check_terminators, {cursor, Sigils, Terminators}}, Acc) ->
+        Acc#elixir_tokenizer{cursor_completion={prune_and_cursor, Sigils}, terminators=Terminators};
       ({existing_atoms_only, ExistingAtomsOnly}, Acc) when is_boolean(ExistingAtomsOnly) ->
         Acc#elixir_tokenizer{existing_atoms_only=ExistingAtomsOnly};
       ({static_atoms_encoder, StaticAtomsEncoder}, Acc) when is_function(StaticAtomsEncoder) ->
@@ -144,12 +144,9 @@ tokenize(String, Line, Opts) ->
 
 tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false ->
   #elixir_tokenizer{ascii_identifiers_only=Ascii, terminators=Terminators, warnings=Warnings} = Scope,
-
-  {CursorColumn, AccTerminators, AccTokens} =
-    add_cursor(Line, Column, Cursor, Terminators, Tokens),
-
+  {CursorColumn, AccTokens} = add_cursor(Line, Column, Cursor, Tokens),
   AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings),
-  {ok, Line, CursorColumn, AllWarnings, AccTokens, AccTerminators};
+  {ok, Line, CursorColumn, AllWarnings, AccTokens, Terminators};
 
 tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) ->
   End = terminator(Start),
@@ -515,7 +512,7 @@ tokenize([$:, H | T] = Original, Line, Column, BaseScope, Tokens) when ?is_quote
   end,
 
   case elixir_interpolation:extract(Line, Column + 2, Scope, true, T, H) of
-    {NewLine, NewColumn, Parts, Rest, InterScope} ->
+    {NewLine, NewColumn, Parts, Rest, _Done, InterScope} ->
       NewScope = case is_unnecessary_quote(Parts, InterScope) of
         true ->
           WarnMsg = io_lib:format(
@@ -788,7 +785,7 @@ handle_char(_)  -> false.
 
 handle_heredocs(T, Line, Column, H, Scope, Tokens) ->
   case extract_heredoc_with_interpolation(Line, Column, Scope, true, T, H) of
-    {ok, NewLine, NewColumn, Parts, Rest, NewScope} ->
+    {ok, NewLine, NewColumn, Parts, Rest, _Done, NewScope} ->
       case unescape_tokens(Parts, Line, Column, NewScope) of
         {ok, Unescaped} ->
           Token = {heredoc_type(H), {Line, Column, nil}, NewColumn - 4, Unescaped},
@@ -807,7 +804,7 @@ handle_strings(T, Line, Column, H, Scope, Tokens) ->
     {error, Reason} ->
       interpolation_error(Reason, [H | T], Scope, Tokens, " (for string starting at line ~B)", [Line], Line, Column-1, [H], [H]);
 
-    {NewLine, NewColumn, Parts, [$: | Rest], InterScope} when ?is_space(hd(Rest)) ->
+    {NewLine, NewColumn, Parts, [$: | Rest], _Done, InterScope} when ?is_space(hd(Rest)) ->
       NewScope = case is_unnecessary_quote(Parts, InterScope) of
         true ->
           WarnMsg = io_lib:format(
@@ -850,7 +847,7 @@ handle_strings(T, Line, Column, H, Scope, Tokens) ->
           error(Reason, Rest, NewScope, Tokens)
       end;
 
-    {NewLine, NewColumn, Parts, Rest, InterScope} ->
+    {NewLine, NewColumn, Parts, Rest, _Done, InterScope} ->
       NewScope =
         case H of
           $' ->
@@ -961,7 +958,7 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, BaseScope, Tokens) whe
   end,
 
   case elixir_interpolation:extract(Line, Column + 1, Scope, true, T, H) of
-    {NewLine, NewColumn, [Part], Rest, InterScope} when is_list(Part) ->
+    {NewLine, NewColumn, [Part], Rest, _Done, InterScope} when is_list(Part) ->
       NewScope = case is_unnecessary_quote([Part], InterScope) of
         true ->
           WarnMsg = io_lib:format(
@@ -992,7 +989,7 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, BaseScope, Tokens) whe
           error(Reason, Original, NewScope, Tokens)
       end;
 
-    {_NewLine, _NewColumn, _Parts, Rest, NewScope} ->
+    {_NewLine, _NewColumn, _Parts, Rest, _Done, NewScope} ->
       Message = "interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: ",
       error({?LOC(Line, Column), Message, [H]}, Rest, NewScope, Tokens);
     {error, Reason} ->
@@ -1149,11 +1146,11 @@ unsafe_to_atom(List, Line, Column, #elixir_tokenizer{}) when is_list(List) ->
       end
   end.
 
-collect_modifiers([H | T], Buffer) when ?is_downcase(H) or ?is_upcase(H) or ?is_digit(H) ->
-  collect_modifiers(T, [H | Buffer]);
+collect_modifiers([H | T], Column, Buffer) when ?is_downcase(H) or ?is_upcase(H) or ?is_digit(H) ->
+  collect_modifiers(T, Column + 1, [H | Buffer]);
 
-collect_modifiers(Rest, Buffer) ->
-  {Rest, lists:reverse(Buffer)}.
+collect_modifiers(Rest, Column, Buffer) ->
+  {Rest, Column, lists:reverse(Buffer)}.
 
 %% Heredocs
 
@@ -1164,14 +1161,14 @@ extract_heredoc_with_interpolation(Line, Column, Scope, Interpol, T, H) ->
       %% spaces later. This new line is removed by calling "tl"
       %% in the final heredoc body three lines below.
       case elixir_interpolation:extract(Line, Column, Scope, Interpol, [$\n|Headerless], [H,H,H]) of
-        {NewLine, NewColumn, Parts0, Rest, InterScope} ->
+        {NewLine, NewColumn, Parts0, Rest, Done, InterScope} ->
           Indent = NewColumn - 4,
           Fun = fun(Part, Acc) -> extract_heredoc_indent(Part, Acc, Indent) end,
           {Parts1, {ShouldWarn, _}} = lists:mapfoldl(Fun, {false, Line}, Parts0),
           Parts2 = extract_heredoc_head(Parts1),
           NewScope = maybe_heredoc_warn(ShouldWarn, Column, InterScope, H),
           try
-            {ok, NewLine, NewColumn, tokens_to_binary(Parts2), Rest, NewScope}
+            {ok, NewLine, NewColumn, tokens_to_binary(Parts2), Rest, Done, NewScope}
           catch
             error:#{'__struct__' := 'Elixir.UnicodeConversionError', message := Message} ->
               {error, interpolation_format(Message, " (for heredoc starting at line ~B)", [Line], Line, Column, [H, H, H], [H, H, H])}
@@ -1746,9 +1743,9 @@ sigil_name_error() ->
 tokenize_sigil_contents([H, H, H | T] = Original, [S | _] = SigilName, Line, Column, Scope, Tokens)
     when ?is_quote(H) ->
   case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of
-    {ok, NewLine, NewColumn, Parts, Rest, NewScope} ->
+    {ok, NewLine, NewColumn, Parts, Rest, Done, NewScope} ->
       Indentation = NewColumn - 4,
-      add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, NewScope, Tokens, Indentation, <<H, H, H>>);
+      add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Done, NewScope, Tokens, Indentation, <<H, H, H>>);
 
     {error, Reason} ->
       error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens)
@@ -1757,9 +1754,10 @@ tokenize_sigil_contents([H, H, H | T] = Original, [S | _] = SigilName, Line, Col
 tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, Scope, Tokens)
     when ?is_sigil(H) ->
   case elixir_interpolation:extract(Line, Column + 1, Scope, ?is_downcase(S), T, sigil_terminator(H)) of
-    {NewLine, NewColumn, Parts, Rest, NewScope} ->
+    {NewLine, NewColumn, Parts, Rest, Done, NewScope} ->
       Indentation = nil,
-      add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, tokens_to_binary(Parts), Rest, NewScope, Tokens, Indentation, <<H>>);
+      add_sigil_token(SigilName, Line, Column, NewLine, NewColumn,
+                      tokens_to_binary(Parts), Rest, Done, NewScope, Tokens, Indentation, <<H>>);
 
     {error, Reason} ->
       Sigil = [$~, S, H],
@@ -1779,7 +1777,7 @@ tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Toke
 tokenize_sigil_contents([], _SigilName, Line, Column, Scope, Tokens) ->
   tokenize([], Line, Column, Scope, Tokens).
 
-add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, Tokens, Indentation, Delimiter) ->
+add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Done, Scope, Tokens, Indentation, Delimiter) ->
   TokenColumn = Column - 1 - length(SigilName),
   MaybeEncoded = case SigilName of
     % Single-letter sigils present no risk of atom exhaustion (limited possibilities)
@@ -1788,9 +1786,12 @@ add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope,
   end,
   case MaybeEncoded of
     {ok, Atom} ->
-      {Final, Modifiers} = collect_modifiers(Rest, []),
-      Token = {sigil, {Line, TokenColumn, nil}, Atom, Parts, Modifiers, Indentation, Delimiter},
-      NewColumnWithModifiers = NewColumn + length(Modifiers),
+      {Final, NewColumnWithModifiers, Modifiers} = case Done of
+        true -> collect_modifiers(Rest, NewColumn, []);
+        false -> {Rest, NewColumn, false}
+      end,
+      Token = {sigil, {Line, TokenColumn, {NewLine, NewColumnWithModifiers}},
+               Atom, Parts, Modifiers, Indentation, Delimiter},
       tokenize(Final, NewLine, NewColumnWithModifiers, Scope, [Token | Tokens]);
 
     {error, Reason} ->
@@ -1903,9 +1904,19 @@ format_error({Location, Error, Token}) ->
 
 %% Cursor handling
 
-add_cursor(_Line, Column, noprune, Terminators, Tokens) ->
-  {Column, Terminators, Tokens};
-add_cursor(Line, Column, prune_and_cursor, Terminators, Tokens) ->
+add_cursor(_Line, Column, noprune, Tokens) ->
+  {Column, Tokens};
+
+add_cursor(Line, Column, {prune_and_cursor, true},
+           [{sigil, {_, _, {Line, Column}} = Location, Name, Parts, Modifier, Identation, Delimiter} | Rest]) ->
+  Cursor = {'__cursor__', [{line, Line}, {column, Column}], []},
+  CursorModifier = case Modifier of
+    false -> Cursor;
+    List -> List ++ [Cursor]
+  end,
+  {Column + 12, [{sigil, Location, Name, Parts, CursorModifier, Identation, Delimiter} | Rest]};
+
+add_cursor(Line, Column, {prune_and_cursor, _}, Tokens) ->
   PrePrunedTokens = prune_identifier(Tokens),
   PrunedTokens = prune_tokens(PrePrunedTokens, []),
   CursorTokens = [
@@ -1914,7 +1925,7 @@ add_cursor(Line, Column, prune_and_cursor, Terminators, Tokens) ->
     {paren_identifier, {Line, Column, nil}, '__cursor__'}
     | PrunedTokens
   ],
-  {Column + 12, Terminators, CursorTokens}.
+  {Column + 12, CursorTokens}.
 
 prune_identifier([{identifier, _, _} | Tokens]) -> Tokens;
 prune_identifier(Tokens) -> Tokens.

lib/elixir/test/elixir/code_fragment_test.exs

@@ -1485,6 +1485,40 @@ defmodule CodeFragmentTest do
       assert cc2q!("foo(123, ~r/") == s2q!("foo(123, __cursor__())")
     end
 
+    test "sigils" do
+      assert cc2q!("foo(123, ~") == s2q!("foo(123, __cursor__())")
+      assert cc2q!("foo(123, ~r") == s2q!("foo(123, __cursor__())")
+      assert cc2q!("foo(123, ~r/") == s2q!("foo(123, __cursor__())")
+      assert cc2q!("foo(123, ~r/foo") == s2q!("foo(123, __cursor__())")
+      assert cc2q!("foo(123, ~r//") == s2q!("foo(123, __cursor__())")
+      assert cc2q!("foo(123, ~r//i") == s2q!("foo(123, __cursor__())")
+      assert cc2q!("foo(123, ~r//i ") == s2q!("foo(123, __cursor__())")
+
+      drop_metadata = fn ast ->
+        Macro.prewalk(ast, fn node ->
+          Macro.update_meta(node, &Keyword.drop(&1, [:delimiter, :column]))
+        end)
+      end
+
+      assert cc2q!("foo(123, ~", preserve_sigils: true) == s2q!("foo(123, __cursor__())")
+      assert cc2q!("foo(123, ~r", preserve_sigils: true) == s2q!("foo(123, __cursor__())")
+
+      assert cc2q!("foo(123, ~r/", preserve_sigils: true) |> drop_metadata.() ==
+               s2q!("foo(123, sigil_r(<<\"\">>, __cursor__()))")
+
+      assert cc2q!("foo(123, ~r/foo", preserve_sigils: true) |> drop_metadata.() ==
+               s2q!("foo(123, sigil_r(<<\"foo\">>, __cursor__()))")
+
+      assert cc2q!("foo(123, ~r//", preserve_sigils: true) |> drop_metadata.() ==
+               s2q!("foo(123, sigil_r(<<\"\">>, [__cursor__()]))")
+
+      assert cc2q!("foo(123, ~r//i", preserve_sigils: true) |> drop_metadata.() ==
+               s2q!("foo(123, sigil_r(<<\"\">>, [?i, __cursor__()]))")
+
+      assert cc2q!("foo(123, ~r//i ", preserve_sigils: true) |> drop_metadata.() ==
+               s2q!("foo(123, __cursor__())")
+    end
+
     test "no warnings" do
       assert cc2q!(~s"?\\ ") == s2q!("__cursor__()")
       assert cc2q!(~s"{fn -> end, ") == s2q!("{fn -> nil end, __cursor__()}")

lib/elixir/test/erlang/string_test.erl

@@ -15,7 +15,7 @@ extract_interpolations(String) ->
   case elixir_interpolation:extract(1, 1, #elixir_tokenizer{}, true, String ++ [$"], $") of
     {error, Error} ->
       Error;
-    {_, _, Parts, _, _} ->
+    {_, _, Parts, _, _, _} ->
       Parts
    end.
 
@@ -26,7 +26,7 @@ extract_interpolations_without_interpolation_test() ->
 
 extract_interpolations_with_escaped_interpolation_test() ->
   ["f\\#{o}o"] = extract_interpolations("f\\#{o}o"),
-  {1, 10, ["f\\#{o}o"], [], _} =
+  {1, 10, ["f\\#{o}o"], [], _, _} =
     elixir_interpolation:extract(1, 2, #elixir_tokenizer{}, true, "f\\#{o}o\"", $").
 
 extract_interpolations_with_interpolation_test() ->

lib/elixir/test/erlang/tokenizer_test.erl

@@ -263,22 +263,22 @@ vc_merge_conflict_test() ->
     tokenize_error("<<<<<<< HEAD\n[1, 2, 3]").
 
 sigil_terminator_test() ->
-  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"),
-  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"),
-  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""),
-  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>},
+  [{sigil, {1, 1, {1, 8}}, sigil_r, [<<"foo">>], [], nil, <<"/">>}] = tokenize("~r/foo/"),
+  [{sigil, {1, 1, {1, 8}}, sigil_r, [<<"foo">>], [], nil, <<"[">>}] = tokenize("~r[foo]"),
+  [{sigil, {1, 1, {1, 8}}, sigil_r, [<<"foo">>], [], nil, <<"\"">>}] = tokenize("~r\"foo\""),
+  [{sigil, {1, 1, {1, 8}}, sigil_r, [<<"foo">>], [], nil, <<"/">>},
    {comp_op, {1, 9, nil}, '=='},
    {identifier, {1, 12, _}, bar}] = tokenize("~r/foo/ == bar"),
-  [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "iu", nil, <<"/">>},
+  [{sigil, {1, 1, {1, 10}}, sigil_r, [<<"foo">>], "iu", nil, <<"/">>},
    {comp_op, {1, 11, nil}, '=='},
    {identifier, {1, 14, _}, bar}] = tokenize("~r/foo/iu == bar"),
-  [{sigil, {1, 1, nil}, sigil_M, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8").
+  [{sigil, {1, 1, {1, 12}}, sigil_M, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8").
 
 sigil_heredoc_test() ->
-  [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),
-  [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"),
-  [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n  sigil heredoc\n  \"\"\""),
-  [{sigil, {1, 1, nil}, sigil_s, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n  sigil heredoc\n  \"\"\"").
+  [{sigil, {1, 1, {3, 4}}, sigil_S, [<<"sigil heredoc\n">>], [], 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),
+  [{sigil, {1, 1, {3, 4}}, sigil_S, [<<"sigil heredoc\n">>], [], 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"),
+  [{sigil, {1, 1, {3, 6}}, sigil_S, [<<"sigil heredoc\n">>], [], 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n  sigil heredoc\n  \"\"\""),
+  [{sigil, {1, 1, {3, 6}}, sigil_s, [<<"sigil heredoc\n">>], [], 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n  sigil heredoc\n  \"\"\"").
 
 invalid_sigil_delimiter_test() ->
   {[{line, 1}, {column, 1}], "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"),