系列入口:编程实战:类C语法的编译型脚本解释器(系列)-CSDN博客

        现在开始解释所有的设计思想和与源代码。先从外围入手,最后进入一个巨大的解析语法的类。

        本文介绍TOKEN和变量

目录

一、TOKEN

1.1 定义Token类型

1.2 将脚本拆解为Token

1.3 TryGetKeyword识别关键字

1.4 TryGetNumber识别数值

1.5 其余TryGetXXXX略 

二、变量


一、TOKEN

1.1 定义Token类型

        token编程语言基本单元,是最小单位,包括分隔符、标识符操作符关键字数字字符串字面值,不包括空白。在C和类C语法中,空白字符包括换行都会被忽略(但预处理程序并非如此,所以预处理额外的东西)。

        所有编译程序首先都会把源代码分解成一系列token,本代码也是如此。token相关定义如下

	enum { TOKEN_BUF_LEN = 128 };//仅用于定义关键字运算符,其它标识符任意长度
	//语法标记,去除空白之后的每个元素
	struct Token
	{
		enum types { DELIMITER = 0, OPERATOR, IDENTIFIER, NUMBER, KEYWORD, STRING };

		types type;//类型
		string text;//文本
		size_t pos;//在源代码中的位置

		Token(types _type, char const* _text, size_t _pos) :type(_type), text(_text), pos(_pos) {}

		string ToString()const
		{
			STATIC_C const char typestr[][TOKEN_BUF_LEN] = { "DELIMITER","OPERATOR","IDENTIFIER","NUMBER","KEYWORD","STRING" };//必须与types对应
			char buf[TOKEN_BUF_LEN * 2];
			string ret;
			sprintf(buf, "%03ld %-12s ", pos, typestr[type]);
			ret = buf;
			ret += text.c_str();
			return ret;
		}
	};

        每个Token包含类型文本对应源代码中的表现形式),同时为了调试需要,增加了pos记录脚本中的位置

1.2 将脚本拆解为Token

        作为编译的第一步,显然是将源代码分解为token

        这一步由一个类来实现

class CTokens
{
		public:
			vector<Token &gt; m_tokens;//解析出的语法元素
			bool ToTokens(string&amp; source)
			{
				string::size_type pos = 0;
				while (GetToken(source, pos));
				return true;
			}
};

        解析出的token保存在m_tokens中,而函数ToTokens()仅仅是循环调用GetToken()解析出一个一个token而已。 

        GetToken()是关键的主控函数

			bool GetToken(string&amp; source, string::size_type&amp; pos)
			{
				Token::types type;
				string token;
				char c;
				bool isInComment = false;
				while (pos < source.size())
				{
					c = source[pos];
					if (isInComment)
					{
						if ('n' == c)
						{
							isInComment = false;
						}
						++pos;
						continue;
					}
					if ('/' == c &amp;&amp; pos + 1 < source.size() &amp;&amp; '/' == source[pos + 1])
					{
						isInComment = true;
						pos += 2;
						continue;
					}
					if (!IsBlank(c))break;
					++pos;
				}
				if (source.size() == pos)return false;
				if (TryGetKeyword(source.c_str(), pos, token))
				{
					type = Token::KEYWORD;
				}
				else if (TryGetNumber(source.c_str(), pos, token))
				{
					type = Token::NUMBER;
				}
				else if (TryGetString(source.c_str(), pos, token))
				{
					type = Token::STRING;
				}
				else if (TryGetDelimiter(source.c_str(), pos, token))
				{
					type = Token::DELIMITER;
				}
				else if (TryGetOperator(source.c_str(), pos, token))
				{
					type = Token::OPERATOR;
				}
				else if (TryGetIdentifier(source.c_str(), pos, token))
				{
					type = Token::IDENTIFIER;
				}
				else
				{
					CmyException::Throw(__FILE__, __LINE__, source.c_str(), pos, "无法识别符号");
					return false;
				}
				m_tokens.push_back(Token(type, token.c_str(), pos - token.size()));
				return true;
			}

        这个函数流程复杂,先跳过注释(仅支持单行注释),然后依次尝试每种token,每种尝试如果成功会修改当前位置pos(通过引用参数),如果失败则不会修改pos。

        TryGetXXXX这一组函数每个都不复杂,不过调用顺序有名堂,关键字是最优先的,这就保证关键字可能被用作变量名

1.3 TryGetKeyword识别关键字

        这个复杂一些,由几个函数组合而成:

			//headset最后一个必须是空串
			bool IsStartWith(char const* str, char const (*headset)[TOKEN_BUF_LEN], string&amp; ret)const
			{
				long i = 0;
				ret = "";
				while (headset[i][0] != '')
				{
					size_t keylen = strlen(headset[i]);
					if (0 == strncmp(headset[i], str, keylen))
					{
						if (ret.size() < strlen(headset[i]))ret = headset[i];
					}
					++i;
				}
				return ret.size() != 0;
			}
			bool IsKeyword(char const* str, string& key)const
			{
				STATIC_C char const buf[][TOKEN_BUF_LEN] = {
					"asm","default","float","operator","static_cast","union",
					"auto","delete","for","private","struct","unsigned",
					"bool","do","friend","protected","switch","using",
					"break","double","goto","public","template","virtual",
					"case","dynamic_cast","if","register","this","void",
					"catch","else","inline","reinterpret_cast","throw","volatile",
					"char","enum","int","return","true","wchar_t",
					"class","explicit","long","short","try","while",
					"const","export","mutable","signed","typedef",
					"const_cast","extern","namespace","sizeof","typeid",
					"continue","false","new","static","typename","string",
					""
				};//必须以空串结尾
				return IsStartWith(str, buf, key);
			}
			bool TryGetKeyword(char const* source, string::size_type& pos, string& ret)
			{
				string key;
				string nextkey;
				size_t keylen;
				if (IsKeyword(source + pos, key))
				{
					keylen = key.size();
					if ('' == source[pos + keylen] || IsBlank(source[pos + keylen]) || IsDelimiter(source[pos + keylen]) || IsOperator(source + pos + keylen, nextkey))
					{
						ret = key;
						pos += keylen;
						return true;
					}
				}
				return false;
			}

         规则其实也很简单:以关键字开头并且其后是{空白分隔符、操作符}则为一个关键字

1.4 TryGetNumber识别数值

        数值是由数字或点开始的字母数字小数点的串,同时还需要符合一些规则代码里分两步进行,第一步识别出串,第二步则将串根据各种规则转换数值

			//以数字或点开头的串
			bool TryGetNumber(char const* source, string::size_type& pos, string& ret)
			{
				ret = "";
				char c = source[pos];
				if (c >= '0' && c <= '9' || c == '.' && source[pos + 1] >= '0' && source[pos + 1] <= '9')
				{
					while ((c = source[pos]) != '')
					{
						if (c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || '.' == c || '_' == c)
						{
						}
						else
						{
							break;
						}
						ret += c;
						++pos;
					}
				}
				return ret.size() != 0;
			}
			bool NumberToVariable(char const* source, Variable& var)
			{
				char* endptr;
				if (IsCharIn('.', source) || IsCharIn('e', source) || IsCharIn('E', source))
				{
					var.type = Variable::DOUBLE;
					var.dValue = strtod(source, &endptr);
				}
				else
				{
					var.type = Variable::LONG;
					long prefix = 0;
					long radix = 10;
					if (strlen(source) >= 1 && '0' == source[0])
					{
						if (strlen(source) >= 2 && ('x' == source[1] || 'X' == source[1]))
						{
							radix = 16;
							prefix = 2;
						}
						else
						{
							radix = 8;
							prefix = 1;
						}
					}
					var.lValue = strtol(source + prefix, &endptr, radix);
				}
				if (strlen(endptr) != 0)
				{
					if (Variable::DOUBLE == var.type && (0 == stricmp(endptr, "f") || 0 == stricmp(endptr, "l"))
						|| Variable::LONG == var.type && (0 == stricmp(endptr, "u") || 0 == stricmp(endptr, "l") || 0 == stricmp(endptr, "i64")))
					{
						return true;
					}
					string str;
					str = "数值常量格式错误 ";
					str += endptr;
					CException::Throw(__FILE__, __LINE__, source, endptr - source, str.c_str());
					return false;
				}
				return true;
			}

1.5 其余TryGetXXXX略 

二、变量

        我非常痛恨无类型变量,比如JavaScript,所以我在这个脚本里面使用类型

        变量类型做了简化分为longdouble和string。通过一个类存储所有的变量,代码如下

	//变量
	struct Variable
	{
		enum types { NULLVARIABLE = 0, LONG, DOUBLE, STRING };
		types type;
		bool isconst;
		long lValue;
		double dValue;
		string strValue;

		Variable() :type(NULLVARIABLE), isconst(false), lValue(0), dValue(0.) {}
		bool isNull() { return type == NULLVARIABLE; }
		bool isNumber() { return type == LONG || type == DOUBLE; }
		bool isString() { return type == STRING; }
		void clear()
		{
			type = NULLVARIABLE;
			isconst = false;
			lValue = 0;
			dValue = 0;
			strValue = "";
		}
		void initvalue()
		{
			lValue = 0;
			dValue = 0;
			strValue = "";
		}
		Variable& operator = (long v)
		{
			char buf[256];
			if (NULLVARIABLE == type)type = LONG;
			switch (type)
			{
			case LONG:lValue = v; break;
			case DOUBLE:dValue = v; break;
			case STRING:
				sprintf(buf, "%ld", v);
				strValue = buf;
				break;
			default:break;
			}
			return *this;
		}
		Variable& operator = (double v)
		{
			char buf[256];
			if (NULLVARIABLE == type)type = DOUBLE;
			switch (type)
			{
			case LONG:lValue = (long)v; break;
			case DOUBLE:dValue = v; break;
			case STRING:
				gcvt(v, 200, buf);
				strValue = buf;
				break;
			default:break;
			}
			return *this;
		}
		Variable& operator = (string const& v)
		{
			if (NULLVARIABLE == type)type = STRING;
			switch (type)
			{
			case LONG:lValue = atol(v.c_str()); break;
			case DOUBLE:dValue = atof(v.c_str()); break;
			case STRING:strValue = v; break;
			default:break;
			}
			return *this;
		}
		Variable& operator = (Variable const& v)
		{
			if (NULLVARIABLE == type)type = v.type;
			switch (type)
			{
			case LONG:lValue = v.GetLong(); break;
			case DOUBLE:dValue = v.GetDouble(); break;
			case STRING:strValue = v.GetString(); break;
			default:break;
			}
			return *this;
		}
		Variable operator-()const
		{
			Variable tmp = *this;
			switch (type)
			{
			case LONG:tmp.lValue = -lValue; break;
			case DOUBLE:tmp.dValue = -dValue; break;
			default:break;
			}
			return tmp;
		}
		//eva=true则是为赋值提升,结果以左边为static types typeUpgrade(types a, types b, bool eva = false)
		{
			if (NULLVARIABLE == a || NULLVARIABLE == b)return NULLVARIABLE;
			if (LONG == a && LONG == b)return LONG;
			if (STRING == a && STRING == b)return STRING;
			if (DOUBLE == a && DOUBLE == b)return DOUBLE;
			if (!eva)
			{
				if (DOUBLE == a && LONG == b)return DOUBLE;
				if (LONG == a && DOUBLE == b)return DOUBLE;
			}
			else
			{
				if (DOUBLE == a && LONG == b)return DOUBLE;
				if (LONG == a && DOUBLE == b)return LONG;
			}
			return NULLVARIABLE;
		}
		long GetLong()const
		{
			string tmp;
			switch (type)
			{
			case LONG: return lValue;
			case DOUBLE: return (long)dValue;
			case STRING: tmp = strValue; return atol(tmp.c_str());
			default:return 0;
			}
		}
		double GetDouble()const
		{
			string tmp;
			switch (type)
			{
			case LONG: return lValue;
			case DOUBLE: return dValue;
			case STRING: tmp = strValue; return atof(tmp.c_str());
			default:return 0.;
			}
		}
		bool GetBool()const
		{
			switch (type)
			{
			case LONG: return 0 != lValue;
			case DOUBLE: return 0 != dValue;
			default:return false;
			}
		}
		string GetString()const
		{
			char buf[256];
			switch (type)
			{
			case LONG: sprintf(buf, "%ld", lValue); return buf;
			case DOUBLE: gcvt(dValue, 200, buf); return buf;
			case STRING: return strValue;
			default:return "";
			}
		}
		Variable operator+(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = typeUpgrade(type, b.type);
			switch (tmp.type)
			{
			case LONG:tmp.lValue = GetLong() + b.GetLong(); break;
			case DOUBLE:tmp.dValue = GetDouble() + b.GetDouble(); break;
			case STRING:tmp.strValue = GetString() + b.GetString(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator-(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = typeUpgrade(type, b.type);
			switch (tmp.type)
			{
			case LONG:tmp.lValue = GetLong() - b.GetLong(); break;
			case DOUBLE:tmp.dValue = GetDouble() - b.GetDouble(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator*(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = typeUpgrade(type, b.type);
			switch (tmp.type)
			{
			case LONG:tmp.lValue = GetLong() * b.GetLong(); break;
			case DOUBLE:tmp.dValue = GetDouble() * b.GetDouble(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator/(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = typeUpgrade(type, b.type);
			switch (tmp.type)
			{
			case LONG:
			{
				if (0 == b.GetLong())throw "div zero";
				tmp.lValue = GetLong() / b.GetLong(); break;
			}
			case DOUBLE:
			{
				if (0 == b.GetDouble())throw "div zero";
				tmp.dValue = GetDouble() / b.GetDouble(); break;
			}
			default:break;
			}
			return tmp;
		}
		Variable operator%(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = typeUpgrade(type, b.type);
			switch (tmp.type)
			{
			case LONG:
			{
				if (0 == b.GetLong())throw "mod zero";
				tmp.lValue = GetLong() % b.GetLong(); break;
			}
			default:break;
			}
			return tmp;
		}
		Variable operator>(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() > b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() > b.GetDouble(); break;
			case STRING:tmp.lValue = GetString() > b.GetString(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator<(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() < b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() < b.GetDouble(); break;
			case STRING:tmp.lValue = GetString() < b.GetString(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator>=(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() >= b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() >= b.GetDouble(); break;
			case STRING:tmp.lValue = GetString() >= b.GetString(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator<=(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() <= b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() <= b.GetDouble(); break;
			case STRING:tmp.lValue = GetString() <= b.GetString(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator==(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() == b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() == b.GetDouble(); break;
			case STRING:tmp.lValue = GetString() == b.GetString(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator!=(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() != b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() != b.GetDouble(); break;
			case STRING:tmp.lValue = GetString() != b.GetString(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator&&(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() && b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() && b.GetDouble(); break;
			default:break;
			}
			return tmp;
		}
		Variable operator||(Variable const& b)const
		{
			Variable tmp = *this;
			tmp.type = LONG;
			switch (typeUpgrade(type, b.type))
			{
			case LONG:tmp.lValue = GetLong() || b.GetLong(); break;
			case DOUBLE:tmp.lValue = GetDouble() || b.GetDouble(); break;
			default:break;
			}
			return tmp;
		}
		static char const* TypeStr(types type)
		{
			STATIC_C const char typestr[][TOKEN_BUF_LEN] = { "NULLVARIABLE","LONG","DOUBLE","STRING" };//必须与types对应
			if(type>=0 && type<4)return typestr[type];
			else
			{
				static char buf[256];
				sprintf(buf, "错误的类型 %d", type);
				//cout << buf << endl; exit(0);
				return buf;
			}
		}
		string ToString(long level = 0)const
		{
			string ret;
			char buf[256];
			string prefix;
			prefix.assign(level * 4, ' ');
			switch (type)
			{
			case LONG:sprintf(buf, "%ld", lValue); break;
			case DOUBLE:gcvt(dValue, 200, buf); break;
			case STRING:strcpy(buf, strValue.c_str()); break;
			default:sprintf(buf, "NULL"); break;
			}
			ret = prefix + " ";
			ret += (isconst ? "常量" : "变量");
			ret += "类型 ";
			ret += TypeStr(type);
			ret += " : ";
			ret += buf;
			return ret;
		}
	};

        没有使用union,直接用类型和三个变量来存储空间当然是有浪费的,但是没人知道啊。

        重载了各种类型的相互操作,都很简单,只是繁琐。

        成员变量:

类型 变量名 功能
enum types type 实际存储的类型
bool isconst 是否是常量常量不允许修改
long lValue type为LONG时使用
double dValue type为DOUBLE时使用
string strValue type为STRING时使用

(待续)

这里结束

原文地址:https://blog.csdn.net/2301_77171572/article/details/134750977

本文来自互联网用户投稿,该文观点仅代表作者本人,不代表本站立场。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任

如若转载,请注明出处:http://www.7code.cn/show_29718.html

如若内容造成侵权/违法违规/事实不符,请联系代码007邮箱suwngjj01@126.com进行投诉反馈,一经查实,立即删除

发表回复

您的邮箱地址不会被公开。 必填项已用 * 标注