enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-11-24 00:27:16 +03:00
Code Issues Projects Releases Wiki Activity

support escaping in raw strings (#369)

Browse Source
This commit is contained in:
Josef 2019-12-05 11:13:46 +01:00 committed by GitHub
parent db4cbacec3
commit 8ae06e0f09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 98 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/AST.scala
View File

@ -7,7 +7,7 @@ import cats.derived._
import cats.implicits._
import io.circe.Encoder
import io.circe.Json
import org.enso.syntax.text.ast.text.Escape
import org.enso.syntax.text.ast.text.{Escape, RawEscape}
import org.enso.data.List1._
import org.enso.data.List1
import org.enso.data.Index
@ -267,6 +267,9 @@ object Shape extends ShapeImplicit {
final case class SegmentEscape[T](code: Escape)
extends SegmentFmt[T]
with Phantom
final case class SegmentRawEscape[T](code: RawEscape)
extends SegmentRaw[T]
with Phantom
///////////
/// App ///
@ -601,32 +604,38 @@ object Shape extends ShapeImplicit {
implicit def ftor[T]: Functor[SegmentFmt] = semi.functor
implicit def fold: Foldable[SegmentFmt] = semi.foldable
implicit def repr[T: Repr]: Repr[SegmentFmt[T]] = {
case t: SegmentPlain[T] => Repr(t)
case t: SegmentExpr[T] => Repr(t)
case t: SegmentEscape[T] => Repr(t)
case t: SegmentPlain[T] => Repr(t)
case t: SegmentExpr[T] => Repr(t)
case t: SegmentEscape[T] => Repr(t)
case t: SegmentRawEscape[T] => Repr(t)
}
implicit def ozip[T]: OffsetZip[SegmentFmt, T] = {
case t: SegmentPlain[T] => OffsetZip(t)
case t: SegmentExpr[T] => OffsetZip(t)
case t: SegmentEscape[T] => OffsetZip(t)
case t: SegmentPlain[T] => OffsetZip(t)
case t: SegmentExpr[T] => OffsetZip(t)
case t: SegmentEscape[T] => OffsetZip(t)
case t: SegmentRawEscape[T] => OffsetZip(t)
}
implicit def span[T: HasSpan]: HasSpan[SegmentFmt[T]] = {
case t: SegmentPlain[T] => t.span()
case t: SegmentExpr[T] => t.span()
case t: SegmentEscape[T] => t.span()
case t: SegmentPlain[T] => t.span()
case t: SegmentExpr[T] => t.span()
case t: SegmentEscape[T] => t.span()
case t: SegmentRawEscape[T] => t.span()
}
}
object SegmentRaw {
implicit def ftor[T]: Functor[SegmentRaw] = semi.functor
implicit def fold: Foldable[SegmentRaw] = semi.foldable
implicit def repr[T]: Repr[SegmentRaw[T]] = {
case t: SegmentPlain[T] => Repr(t)
case t: SegmentPlain[T] => Repr(t)
case t: SegmentRawEscape[T] => Repr(t)
}
implicit def ozip[T]: OffsetZip[SegmentRaw, T] = {
case t: SegmentPlain[T] => OffsetZip(t)
case t: SegmentPlain[T] => OffsetZip(t)
case t: SegmentRawEscape[T] => OffsetZip(t)
}
implicit def span[T]: HasSpan[SegmentRaw[T]] = {
case t: SegmentPlain[T] => t.span()
case t: SegmentPlain[T] => t.span()
case t: SegmentRawEscape[T] => t.span()
}
}
object SegmentPlain {
@ -657,13 +666,25 @@ object Shape extends ShapeImplicit {
implicit def ftor: Functor[SegmentEscape] = semi.functor
implicit def fold: Foldable[SegmentEscape] = semi.foldable
implicit def repr[T: Repr]: Repr[SegmentEscape[T]] =
implicit def repr[T]: Repr[SegmentEscape[T]] =
t => introducer + t.code.repr
implicit def ozip[T]: OffsetZip[SegmentEscape, T] =
t => t.coerce
implicit def span[T: HasSpan]: HasSpan[SegmentEscape[T]] =
introducer.span + _.code.repr.length
}
object SegmentRawEscape {
val introducer: Repr.Builder = "\\"
implicit def ftor: Functor[SegmentRawEscape] = semi.functor
implicit def fold: Foldable[SegmentRawEscape] = semi.foldable
implicit def repr[T]: Repr[SegmentRawEscape[T]] =
t => introducer + t.code.repr
implicit def ozip[T]: OffsetZip[SegmentRawEscape, T] =
t => t.coerce
implicit def span[T]: HasSpan[SegmentRawEscape[T]] =
introducer.span + _.code.repr.length
}
object App extends IntermediateTrait[App] {
implicit def ftor[T]: Functor[App] = semi.functor
implicit def fold: Foldable[App] = semi.foldable
@ -876,7 +897,7 @@ object Shape extends ShapeImplicit {
object Segment {
def apply(head: AST): Segment = Segment(head, None)
implicit def repr: Repr[Segment] = t => R + t.head + t.body
implicit def span: HasSpan[Segment] = t => t.head.span() + t.body.span()
implicit def span: HasSpan[Segment] = t => t.head.span + t.body.span()
}
}
@ -1700,6 +1721,7 @@ object AST {
val Escape = org.enso.syntax.text.ast.text.Escape
type Escape = org.enso.syntax.text.ast.text.Escape
type RawEscape = org.enso.syntax.text.ast.text.RawEscape
//// Definition ////

17
common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/ast/text/Escape.scala
View File

@ -6,11 +6,18 @@ import org.enso.syntax.text.ast.text.Escape.Slash.toString
sealed trait Escape {
val repr: String
}
sealed trait RawEscape {
val repr: String
}
object Escape {
final case class Invalid(str: String) extends Escape {
val repr = str
final case object Unfinished extends RawEscape {
val repr = ""
}
final case class Invalid(str: Char) extends RawEscape {
val repr = str.toString
}
final case class Number(int: Int) extends Escape {
@ -105,17 +112,17 @@ object Escape {
}
}
case object Slash extends Escape {
case object Slash extends RawEscape {
val code: Int = '\\'
def name: String = toString
override val repr = "\\"
}
case object Quote extends Escape {
case object Quote extends RawEscape {
val code: Int = '\''
def name: String = toString
override val repr = "\'"
}
case object RawQuote extends Escape {
case object RawQuote extends RawEscape {
val code: Int = '"'
def name: String = toString
override val repr = "\""

70
common/scala/syntax/definition/src/main/scala/org/enso/syntax/text/spec/ParserDef.scala
View File

@ -1,6 +1,5 @@
package org.enso.syntax.text.spec
import org.enso.data.List1
import org.enso.flexer
import org.enso.flexer.Reader
import org.enso.flexer.State
@ -343,16 +342,19 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
finish(t => AST.Text.Raw(t.head.text: _*), t => AST.Text(t.head.text: _*))
}
def submitUnclosed(): Unit = logger.trace {
def submitMissingQuote(): Unit = logger.trace {
rewind()
val Text = AST.Text.Unclosed
finish(t => Text.Raw(t.head.text: _*), t => Text(t.head.text: _*))
submitUnclosed()
}
def submitDoubleQuote(): Unit = logger.trace {
def submitInvalidQuote(): Unit = logger.trace {
submitUnclosed()
onInvalidQuote()
}
def submitUnclosed(): Unit = logger.trace {
val Text = AST.Text.Unclosed
finish(t => Text.Raw(t.head.text: _*), t => Text(t.head.text: _*))
onInvalidQuote()
}
def onEndOfBlock(): Unit = logger.trace {
@ -403,6 +405,10 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
submit(Shape.SegmentEscape(code))
}
def onEscape(code: Segment.RawEscape): Unit = logger.trace {
submit(Shape.SegmentRawEscape(code))
}
def onEscapeU16(): Unit = logger.trace {
val code = currentMatch.drop(2)
onEscape(Segment.Escape.Unicode.U16(code))
@ -418,9 +424,13 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
onEscape(Segment.Escape.Number(int))
}
def onInvalidEscape(): Unit = logger.trace {
val str = currentMatch.drop(1)
onEscape(Segment.Escape.Invalid(str))
def onEscapeInvalid(): Unit = logger.trace {
val chr = currentMatch.charAt(1)
onEscape(Segment.Escape.Invalid(chr))
}
def onEscapeUnfinished(): Unit = logger.trace {
onEscape(Segment.Escape.Unfinished)
}
def onEscapeSlash(): Unit = logger.trace {
@ -453,11 +463,6 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
}
}
def onTextEOF(): Unit = logger.trace {
submitUnclosed()
rewind()
}
def submitLine(): Unit = logger.trace {
if (state.current == FMT_LINE || state.current == RAW_LINE || text.lineBuilder.nonEmpty) {
val Line = Shape.TextBlockLine
@ -504,7 +509,7 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
val escape_u16 = "\\u" >> repeat(fmtChar, 0, 4)
val escape_u32 = "\\U" >> repeat(fmtChar, 0, 8)
val fmtSeg = fmtChar.many1
val rawSeg = noneOf("\"\n").many1
val rawSeg = noneOf("\"\n\\").many1
val fmtBSeg = noneOf("\n\\`").many1
val rawBSeg = noneOf("\n").many1
@ -528,11 +533,10 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
ROOT || "'" || text.onBegin(text.FMT_LINE)
text.FMT_LINE || "'" || text.submit()
text.FMT_LINE || "''" || text.submitDoubleQuote()
text.FMT_LINE || "'".many1 || text.submitUnclosed()
text.FMT_LINE || "'".many1 || text.submitInvalidQuote()
text.FMT_LINE || text.fmtSeg || text.submitPlainSegment()
text.FMT_LINE || eof || text.onTextEOF()
text.FMT_LINE || newline || text.submitUnclosed()
text.FMT_LINE || eof || text.submitMissingQuote()
text.FMT_LINE || newline || text.submitMissingQuote()
block.FIRSTCHAR || text.fmtBlock || text.onBeginBlock(text.FMT_BLCK)
ROOT || text.fmtBlock || text.onBeginBlock(text.FMT_BLCK)
ROOT || "'''" || text.onInlineBlock()
@ -542,11 +546,10 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
ROOT || '"' || text.onBegin(text.RAW_LINE)
text.RAW_LINE || '"' || text.submit()
text.RAW_LINE || "\"\"" || text.submitDoubleQuote()
text.RAW_LINE || '"'.many1 || text.submitUnclosed()
text.RAW_LINE || '"'.many1 || text.submitInvalidQuote()
text.RAW_LINE || text.rawSeg || text.submitPlainSegment()
text.RAW_LINE || eof || text.onTextEOF()
text.RAW_LINE || newline || text.submitUnclosed()
text.RAW_LINE || eof || text.submitMissingQuote()
text.RAW_LINE || newline || text.submitMissingQuote()
block.FIRSTCHAR || text.rawBlock || text.onBeginBlock(text.RAW_BLCK)
ROOT || text.rawBlock || text.onBeginBlock(text.RAW_BLCK)
ROOT || "\"\"\"" || text.onInlineBlock()
@ -568,14 +571,19 @@ case class ParserDef() extends flexer.Parser[AST.Module] {
text.FMT || s"\\$code" run s"text.onEscape($ctrl)"
}
text.FMT || text.escape_u16 || text.onEscapeU16()
text.FMT || text.escape_u32 || text.onEscapeU32()
text.FMT || text.escape_int || text.onEscapeInt()
text.FMT || "\\\\" || text.onEscapeSlash()
text.FMT || "\\'" || text.onEscapeQuote()
text.FMT || "\\\"" || text.onEscapeRawQuote()
text.FMT || ("\\" >> text.fmtChar) || text.onInvalidEscape()
text.FMT || "\\" || text.submitPlainSegment()
text.FMT || text.escape_u16 || text.onEscapeU16()
text.FMT || text.escape_u32 || text.onEscapeU32()
text.FMT || text.escape_int || text.onEscapeInt()
text.FMT || "\\\\" || text.onEscapeSlash()
text.FMT || "\\'" || text.onEscapeQuote()
text.FMT || "\\" >> any || text.onEscapeInvalid()
text.FMT || "\\" || text.onEscapeUnfinished()
text.RAW_LINE || "\\\"" || text.onEscapeRawQuote()
text.RAW_LINE || "\\\\" || text.onEscapeSlash()
text.RAW_LINE || "\\" >> any || text.onEscapeInvalid()
text.RAW_LINE || "\\" || text.onEscapeUnfinished()
//////////////
/// Blocks ///

16
common/scala/syntax/specialization/src/test/scala/org/enso/syntax/text/ParserTest.scala
View File

@ -219,8 +219,8 @@ class ParserTest extends FlatSpec with Matchers {
//// Escapes ////
val Esc = Text.Segment.Escape
def escape(esc: Text.Segment.Escape): Text.Segment.Fmt =
Shape.SegmentEscape(esc)
def escape(code: Text.Segment.Escape) = Shape.SegmentEscape[AST](code)
def escape(code: Text.Segment.RawEscape) = Shape.SegmentRawEscape[AST](code)
Text.Segment.Escape.Character.codes.foreach(
i => s"'\\$i'" ?= Text(escape(i))
@ -231,12 +231,16 @@ class ParserTest extends FlatSpec with Matchers {
"'\\\\'" ?= Text(escape(Esc.Slash))
"'\\''" ?= Text(escape(Esc.Quote))
"'\\\"'" ?= Text(escape(Esc.RawQuote))
"'\\" ?= Text.Unclosed("\\")
"'\\c'" ?= Text(escape(Esc.Invalid("c")))
"'\\cd'" ?= Text(escape(Esc.Invalid("c")), "d")
"'\\" ?= Text.Unclosed(escape(Esc.Unfinished))
"'\\c'" ?= Text(escape(Esc.Invalid('c')))
"'\\cd'" ?= Text(escape(Esc.Invalid('c')), "d")
"'\\123d'" ?= Text(escape(Esc.Number(123)), "d")
"\"\\\\\"" ?= Text.Raw(escape(Esc.Slash))
"\"\\\"\"" ?= Text.Raw(escape(Esc.RawQuote))
"\"\\" ?= Text.Unclosed.Raw(escape(Esc.Unfinished))
"\"\\cd\"" ?= Text.Raw(escape(Esc.Invalid('c')), "d")
//// Interpolation ////
def expr(ast: AST) = Shape.SegmentExpr(Some(ast))