feat: image and link parser (#1744)

* feat: image and link parser

* chore: update
This commit is contained in:
boojack 2023-05-26 08:43:37 +08:00 committed by GitHub
parent 523ef2bba5
commit dbc85fe7e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 252 additions and 18 deletions

View File

@ -13,11 +13,11 @@ func TestHeadingParser(t *testing.T) {
heading *HeadingParser heading *HeadingParser
}{ }{
{ {
text: "*Hello world!", text: "*Hello world",
heading: nil, heading: nil,
}, },
{ {
text: "## Hello World!", text: "## Hello World",
heading: &HeadingParser{ heading: &HeadingParser{
Level: 2, Level: 2,
ContentTokens: []*tokenizer.Token{ ContentTokens: []*tokenizer.Token{
@ -31,7 +31,7 @@ func TestHeadingParser(t *testing.T) {
}, },
{ {
Type: tokenizer.Text, Type: tokenizer.Text,
Value: "World!", Value: "World",
}, },
}, },
}, },
@ -65,12 +65,12 @@ func TestHeadingParser(t *testing.T) {
}, },
}, },
{ {
text: " # 123123 Hello World!", text: " # 123123 Hello World",
heading: nil, heading: nil,
}, },
{ {
text: `# 123 text: `# 123
Hello World!`, Hello World`,
heading: &HeadingParser{ heading: &HeadingParser{
Level: 1, Level: 1,
ContentTokens: []*tokenizer.Token{ ContentTokens: []*tokenizer.Token{

View File

@ -0,0 +1,55 @@
package parser
import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
type ImageParser struct {
AltText string
URL string
}
func NewImageParser() *ImageParser {
return &ImageParser{}
}
func (*ImageParser) Match(tokens []*tokenizer.Token) *ImageParser {
if len(tokens) < 5 {
return nil
}
if tokens[0].Type != tokenizer.ExclamationMark {
return nil
}
if tokens[1].Type != tokenizer.LeftSquareBracket {
return nil
}
cursor, altText := 2, ""
for ; cursor < len(tokens)-2; cursor++ {
if tokens[cursor].Type == tokenizer.Newline {
return nil
}
if tokens[cursor].Type == tokenizer.RightSquareBracket {
break
}
altText += tokens[cursor].Value
}
if tokens[cursor+1].Type != tokenizer.LeftParenthesis {
return nil
}
matched, url := false, ""
for _, token := range tokens[cursor+2:] {
if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {
return nil
}
if token.Type == tokenizer.RightParenthesis {
matched = true
break
}
url += token.Value
}
if !matched || url == "" {
return nil
}
return &ImageParser{
AltText: altText,
URL: url,
}
}

View File

@ -0,0 +1,42 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestImageParser(t *testing.T) {
tests := []struct {
text string
image *ImageParser
}{
{
text: "![](https://example.com)",
image: &ImageParser{
AltText: "",
URL: "https://example.com",
},
},
{
text: "! [](https://example.com)",
image: nil,
},
{
text: "![alte]( htt ps :/ /example.com)",
image: nil,
},
{
text: "![al te](https://example.com)",
image: &ImageParser{
AltText: "al te",
URL: "https://example.com",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.image, NewImageParser().Match(tokens))
}
}

View File

@ -0,0 +1,58 @@
package parser
import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
type LinkParser struct {
ContentTokens []*tokenizer.Token
URL string
}
func NewLinkParser() *LinkParser {
return &LinkParser{}
}
func (*LinkParser) Match(tokens []*tokenizer.Token) *LinkParser {
if len(tokens) < 4 {
return nil
}
if tokens[0].Type != tokenizer.LeftSquareBracket {
return nil
}
cursor, contentTokens := 1, []*tokenizer.Token{}
for ; cursor < len(tokens)-2; cursor++ {
if tokens[cursor].Type == tokenizer.Newline {
return nil
}
if tokens[cursor].Type == tokenizer.RightSquareBracket {
break
}
contentTokens = append(contentTokens, tokens[cursor])
}
if tokens[cursor+1].Type != tokenizer.LeftParenthesis {
return nil
}
matched, url := false, ""
for _, token := range tokens[cursor+2:] {
if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {
return nil
}
if token.Type == tokenizer.RightParenthesis {
matched = true
break
}
url += token.Value
}
if !matched || url == "" {
return nil
}
if len(contentTokens) == 0 {
contentTokens = append(contentTokens, &tokenizer.Token{
Type: tokenizer.Text,
Value: url,
})
}
return &LinkParser{
ContentTokens: contentTokens,
URL: url,
}
}

View File

@ -0,0 +1,60 @@
package parser
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestLinkParser(t *testing.T) {
tests := []struct {
text string
link *LinkParser
}{
{
text: "[](https://example.com)",
link: &LinkParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "https://example.com",
},
},
URL: "https://example.com",
},
},
{
text: "! [](https://example.com)",
link: nil,
},
{
text: "[alte]( htt ps :/ /example.com)",
link: nil,
},
{
text: "[hello world](https://example.com)",
link: &LinkParser{
ContentTokens: []*tokenizer.Token{
{
Type: tokenizer.Text,
Value: "hello",
},
{
Type: tokenizer.Space,
Value: " ",
},
{
Type: tokenizer.Text,
Value: "world",
},
},
URL: "https://example.com",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
require.Equal(t, test.link, NewLinkParser().Match(tokens))
}
}

View File

@ -17,7 +17,7 @@ func TestParagraphParser(t *testing.T) {
paragraph: nil, paragraph: nil,
}, },
{ {
text: "Hello world!", text: "Hello world",
paragraph: &ParagraphParser{ paragraph: &ParagraphParser{
ContentTokens: []*tokenizer.Token{ ContentTokens: []*tokenizer.Token{
{ {
@ -30,14 +30,14 @@ func TestParagraphParser(t *testing.T) {
}, },
{ {
Type: tokenizer.Text, Type: tokenizer.Text,
Value: "world!", Value: "world",
}, },
}, },
}, },
}, },
{ {
text: `Hello text: `Hello
world!`, world`,
paragraph: &ParagraphParser{ paragraph: &ParagraphParser{
ContentTokens: []*tokenizer.Token{ ContentTokens: []*tokenizer.Token{
{ {
@ -53,7 +53,7 @@ world!`,
}, },
{ {
text: `Hello \n text: `Hello \n
world!`, world`,
paragraph: &ParagraphParser{ paragraph: &ParagraphParser{
ContentTokens: []*tokenizer.Token{ ContentTokens: []*tokenizer.Token{
{ {

View File

@ -3,12 +3,17 @@ package tokenizer
type TokenType = string type TokenType = string
const ( const (
Underline TokenType = "_" Underline TokenType = "_"
Star TokenType = "*" Star TokenType = "*"
Hash TokenType = "#" Hash TokenType = "#"
Backtick TokenType = "`" Backtick TokenType = "`"
Newline TokenType = "\n" LeftSquareBracket TokenType = "["
Space TokenType = " " RightSquareBracket TokenType = "]"
LeftParenthesis TokenType = "("
RightParenthesis TokenType = ")"
ExclamationMark TokenType = "!"
Newline TokenType = "\n"
Space TokenType = " "
) )
const ( const (
@ -37,10 +42,20 @@ func Tokenize(text string) []*Token {
tokens = append(tokens, NewToken(Star, "*")) tokens = append(tokens, NewToken(Star, "*"))
case '#': case '#':
tokens = append(tokens, NewToken(Hash, "#")) tokens = append(tokens, NewToken(Hash, "#"))
case '\n':
tokens = append(tokens, NewToken(Newline, "\n"))
case '`': case '`':
tokens = append(tokens, NewToken(Backtick, "`")) tokens = append(tokens, NewToken(Backtick, "`"))
case '[':
tokens = append(tokens, NewToken(LeftSquareBracket, "["))
case ']':
tokens = append(tokens, NewToken(RightSquareBracket, "]"))
case '(':
tokens = append(tokens, NewToken(LeftParenthesis, "("))
case ')':
tokens = append(tokens, NewToken(RightParenthesis, ")"))
case '!':
tokens = append(tokens, NewToken(ExclamationMark, "!"))
case '\n':
tokens = append(tokens, NewToken(Newline, "\n"))
case ' ': case ' ':
tokens = append(tokens, NewToken(Space, " ")) tokens = append(tokens, NewToken(Space, " "))
default: default:

View File

@ -28,7 +28,11 @@ func TestTokenize(t *testing.T) {
}, },
{ {
Type: Text, Type: Text,
Value: "world!", Value: "world",
},
{
Type: ExclamationMark,
Value: "!",
}, },
}, },
}, },