TokenFactory和Tokenizer基础

src/Token/Factory/DefaultFactory.php

@@ -0,0 +1,32 @@
+<?php
+/**
+ * @filename DefaultFactory.php
+ * @author Jerry Yan <792602257@qq.com>
+ * @date 2020/12/17 14:48
+ */
+
+
+namespace JerryYan\DSL\Token\Factory;
+
+
+use JerryYan\DSL\Token\Token;
+use JerryYan\DSL\Token\TokenAnd;
+use JerryYan\DSL\Token\TokenOr;
+use JerryYan\DSL\Token\TokenVar;
+
+class DefaultFactory extends FactoryInterface
+{
+    protected $tokenMap = [
+        Token::AND => TokenAnd::class,
+        Token::OR => TokenOr::class,
+        Token::VAR => TokenVar::class,
+    ];
+
+    protected $tokenNameMap = [
+        "和" => Token::AND,
+        "或者" => Token::OR,
+        "或" => Token::OR,
+    ];
+
+    protected $undefinedTokenClass = TokenVar::class;
+}

src/Token/Factory/FactoryInterface.php

@@ -0,0 +1,38 @@
+<?php
+/**
+ * @filename FactoryInterface.php
+ * @author Jerry Yan <792602257@qq.com>
+ * @date 2021/1/22 13:42
+ */
+
+
+namespace JerryYan\DSL\Token\Factory;
+
+
+
+use JerryYan\DSL\Token\TokenInterface;
+use JerryYan\DSL\Token\TokenUndefined;
+
+abstract class FactoryInterface
+{
+    /** @var array<string, \JerryYan\DSL\Token\TokenInterface> Token类型及映射类 */
+    protected $tokenMap = [];
+    /** @var array<string, string> Token别名映射 */
+    protected $tokenNameMap = [];
+    /** @var \JerryYan\DSL\Token\TokenInterface 默认Token类 */
+    protected $undefinedTokenClass = TokenUndefined::class;
+
+    public function getTokenByName(string $name): TokenInterface
+    {
+        $originalName = $name;
+        if (isset($this->tokenNameMap[$name])) {
+            $name = $this->tokenNameMap[$name];
+        }
+        if (!isset($this->tokenMap[$name])) {
+            return new $this->undefinedTokenClass($originalName);
+        } else {
+            return new $this->tokenMap[$name]($originalName);
+        }
+    }
+
+}

src/Token/TokenFactory.php

@@ -1,37 +0,0 @@
-<?php
-/**
- * @filename TokenFactory.php
- * @author Jerry Yan <792602257@qq.com>
- * @date 2020/12/17 14:48
- */
-
-
-namespace JerryYan\DSL\Token;
-
-
-class TokenFactory
-{
-    /** @var array<string, string> Token类型及映射类 */
-    private $tokenMap = [
-        Token::AND => TokenAnd::class,
-        Token::OR => TokenOr::class,
-        Token::VAR => TokenVar::class,
-    ];
-
-    protected $tokenNameMap = [
-
-    ];
-
-    public function getTokenByName(string $name): TokenInterface
-    {
-        $originalName = $name;
-        if (isset($this->tokenNameMap[$name])) {
-            $name = $this->tokenNameMap[$name];
-        }
-        if (!isset($this->tokenMap[$name])) {
-            return new TokenUndefined($originalName);
-        } else {
-            return new $this->tokenMap[$name]($originalName);
-        }
-    }
-}

src/Token/TokenInterface.php

@@ -40,12 +40,12 @@ abstract class TokenInterface
      */
     public function getFirstToken(): ?TokenInterface
     {
-        if ($this->hasPrevToken()) return $this->prevToken->getFirstToken();
+        if ($this->hasPrevToken()) return $this->getPrevToken()->getFirstToken();
         else return $this;
     }
     public function getLastToken(): ?TokenInterface
     {
-        if ($this->hasNextToken()) return $this->nextToken->getLastToken();
+        if ($this->hasNextToken()) return $this->getNextToken()->getLastToken();
         else return $this;
     }
     public function hasPrevToken(): bool
@@ -62,4 +62,22 @@ abstract class TokenInterface
     public function getNextToken(): ?TokenInterface{
         return $this->nextToken;
     }
+    public function getPrevTokenLength(): int
+    {
+        if ($this->hasPrevToken()) return $this->getPrevToken()->getPrevTokenLength() + 1;
+        else return 0;
+    }
+    public function getNextTokenLength(): int
+    {
+        if ($this->hasNextToken()) return $this->getNextToken()->getNextTokenLength() + 1;
+        else return 0;
+    }
+    public function getLength(): int
+    {
+        return $this->getPrevTokenLength() + $this->getNextTokenLength() + 1;
+    }
+    public function count(): int
+    {
+        return $this->getLength();
+    }
 }

src/Tokenizer/Tokenizer.php

@@ -10,13 +10,14 @@ namespace JerryYan\DSL\Tokenizer;
 
 
 use JerryYan\DSL\Reader\ReaderInterface;
-use JerryYan\DSL\Token\TokenFactory;
+use JerryYan\DSL\Token\Factory\FactoryInterface;
 use JerryYan\DSL\Token\TokenInterface;
 
 class Tokenizer extends TokenizerInterface
 {
-
-    public function __construct(TokenFactory $tokenFactory)
+    /** @var FactoryInterface token工厂 */
+    protected $tokenFactory;
+    public function __construct(FactoryInterface $tokenFactory)
     {
         $this->tokenFactory = $tokenFactory;
     }

tests/Token/TokenInterfaceTest.php

@@ -0,0 +1,96 @@
+<?php
+/**
+ * @filename TokenInterfaceTest.php
+ * @author Jerry Yan <792602257@qq.com>
+ * @date 2021/1/22 13:52
+ */
+
+namespace JerryYan\DSL\Test\Token;
+
+use JerryYan\DSL\Token\TokenAnd;
+use JerryYan\DSL\Token\TokenDefine;
+use JerryYan\DSL\Token\TokenInterface;
+use JerryYan\DSL\Token\TokenOr;
+use JerryYan\DSL\Token\TokenUndefined;
+use JerryYan\DSL\Token\TokenVar;
+use PHPUnit\Framework\TestCase;
+
+class TokenInterfaceTest extends TestCase
+{
+    /** @var TokenInterface[] TokenClass */
+    private $tokenTypes = [
+        TokenAnd::class,
+        TokenDefine::class,
+        TokenOr::class,
+        TokenUndefined::class,
+        TokenVar::class,
+    ];
+    protected $chainFirst;
+    protected $chainLast;
+    protected function setUp(): void
+    {
+        /** @var ?TokenInterface $next */
+        $last = NULL;
+        foreach ($this->tokenTypes as $cls) {
+            /** @var TokenInterface $current */
+            $current = new $cls('');
+            $current->setPrevToken($last);
+            if ($last !== NULL) $last->setNextToken($current);
+            $last = $current;
+        }
+        $this->chainLast = $last;
+        $this->chainFirst = $last->getFirstToken();
+    }
+
+    public function testGetFirstToken()
+    {
+        $this->assertInstanceOf($this->tokenTypes[0], $this->chainLast->getFirstToken(), '链表头类型非预期');
+    }
+
+    public function testGetPrevToken()
+    {
+        $this->assertInstanceOf($this->tokenTypes[count($this->tokenTypes)-2], $this->chainLast->getPrevToken(), '链表尾前一类型非预期');
+        $this->assertNull($this->chainFirst->getPrevToken(), '链表头前一类型非空');
+    }
+
+    public function testCount()
+    {
+        $this->assertEquals($this->chainFirst->count(), $this->chainLast->count(), '链表头尾计数不同');
+        $this->assertEquals(count($this->tokenTypes), $this->chainLast->count(), '链表尾的长度不同');
+    }
+
+    public function testGetNextToken()
+    {
+        $this->assertInstanceOf($this->tokenTypes[1], $this->chainFirst->getNextToken(), '链表头后一类型非预期');
+        $this->assertNull($this->chainLast->getNextToken(), '链表尾前一类型非空');
+    }
+
+    public function testGetLastToken()
+    {
+        $this->assertInstanceOf($this->tokenTypes[count($this->tokenTypes)-1], $this->chainFirst->getLastToken(), '链表尾类型非预期');
+    }
+
+    public function testHasPrevToken()
+    {
+        $this->assertTrue($this->chainLast->hasPrevToken());
+        $this->assertFalse($this->chainFirst->hasPrevToken());
+    }
+
+    public function testHasNextToken()
+    {
+        $this->assertTrue($this->chainFirst->hasNextToken());
+        $this->assertFalse($this->chainLast->hasNextToken());
+    }
+
+    public function testGetNextTokenLength()
+    {
+        $this->assertEquals(count($this->tokenTypes) - 2, $this->chainFirst->getNextToken()->getNextTokenLength(), '链表头后一后长度不对');
+        $this->assertEquals(0, $this->chainLast->getNextTokenLength(), '链表尾后长度不对');
+    }
+
+    public function testGetPrevTokenLength()
+    {
+        $this->assertEquals(count($this->tokenTypes) - 2, $this->chainLast->getPrevToken()->getPrevTokenLength(), '链表尾前一前长度不对');
+        $this->assertEquals(0, $this->chainFirst->getPrevTokenLength(), '链表头前长度不对');
+    }
+}

tests/Tokenizer/TokenizerTest.php

@@ -0,0 +1,50 @@
+<?php
+/**
+ * @filename TokenizerTest.php
+ * @author Jerry Yan <792602257@qq.com>
+ * @date 2021/1/22 13:33
+ */
+
+namespace JerryYan\DSL\Test\Tokenizer;
+
+use JerryYan\DSL\Reader\StringReader;
+use JerryYan\DSL\Token\Factory\DefaultFactory;
+use JerryYan\DSL\Token\TokenAnd;
+use JerryYan\DSL\Token\TokenInterface;
+use JerryYan\DSL\Token\TokenOr;
+use JerryYan\DSL\Token\TokenVar;
+use JerryYan\DSL\Tokenizer\Tokenizer;
+use PHPUnit\Framework\TestCase;
+
+class TokenizerTest extends TestCase
+{
+    protected $tokenizer;
+    protected $reader;
+    private $text = "这个 和 那个 或者 那个 和 这个";
+    private $textTokenType = [
+        TokenVar::class,
+        TokenAnd::class,
+        TokenVar::class,
+        TokenOr::class,
+        TokenVar::class,
+        TokenAnd::class,
+        TokenVar::class,
+    ];
+    protected function setUp(): void
+    {
+        $this->tokenizer = new Tokenizer(new DefaultFactory());
+        $this->reader = new StringReader($this->text);
+    }
+
+    public function testTokenize()
+    {
+        $tokens = $this->tokenizer->tokenize($this->reader);
+        $this->assertInstanceOf(TokenInterface::class, $tokens);
+        $index = 0;
+        do {
+            $this->assertInstanceOf($this->textTokenType[$index], $tokens);
+            $tokens = $tokens->getNextToken();
+            $index ++;
+        } while ($tokens->hasNextToken());
+    }
+}