Added:
trunk/MySql.Data/Provider/Source/common/MySqlTokenizer.cs
trunk/MySql.Data/Tests/Source/SqlTokenizer.cs
trunk/MySql.Data/Tests/Source/Tokenizer.cs
Removed:
trunk/MySql.Data/Provider/Source/common/SqlTokenizer.cs
Modified:
trunk/MySql.Data/Provider/MySql.Data.CF.csproj
trunk/MySql.Data/Provider/MySql.Data.csproj
trunk/MySql.Data/Provider/Source/ISSchemaProvider.cs
trunk/MySql.Data/Provider/Source/MySqlScript.cs
trunk/MySql.Data/Provider/Source/PreparableStatement.cs
trunk/MySql.Data/Provider/Source/SchemaProvider.cs
trunk/MySql.Data/Provider/Source/Statement.cs
trunk/MySql.Data/Provider/Source/command.cs
trunk/MySql.Data/Tests/MySql.Data.Tests.csproj
Log:
integrated new sql tokenizer and removed the old one. It supports all comment styles and is approx 40% faster
Modified: trunk/MySql.Data/Provider/MySql.Data.CF.csproj
===================================================================
--- trunk/MySql.Data/Provider/MySql.Data.CF.csproj 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/MySql.Data.CF.csproj 2008-08-18 21:41:06 UTC (rev 1383)
@@ -2,7 +2,7 @@
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
- <ProductVersion>9.0.21022</ProductVersion>
+ <ProductVersion>9.0.30729</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{587A47FB-C1CC-459D-93B6-179D95E41EFB}</ProjectGuid>
<OutputType>Library</OutputType>
@@ -74,11 +74,11 @@
</Compile>
<Compile Include="Source\common\Cache.cs" />
<Compile Include="Source\common\ContextString.cs" />
+ <Compile Include="Source\common\MySqlTokenizer.cs" />
<Compile Include="Source\common\NativeMethods.cs" />
<Compile Include="Source\common\Platform.cs" />
<Compile Include="Source\common\SHA1.cs" />
<Compile Include="Source\common\SocketStream.cs" />
- <Compile Include="Source\common\SqlTokenizer.cs" />
<Compile Include="Source\common\StreamCreator.cs" />
<Compile Include="Source\common\Version.cs" />
<Compile Include="Source\CompressedStream.cs" />
Modified: trunk/MySql.Data/Provider/MySql.Data.csproj
===================================================================
--- trunk/MySql.Data/Provider/MySql.Data.csproj 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/MySql.Data.csproj 2008-08-18 21:41:06 UTC (rev 1383)
@@ -219,7 +219,7 @@
<DependentUpon>Resources.resx</DependentUpon>
</Compile>
<Compile Include="Source\common\Cache.cs" />
- <Compile Include="Source\common\SqlTokenizer.cs" />
+ <Compile Include="Source\common\MySqlTokenizer.cs" />
<Compile Include="Source\BulkLoader.cs" />
<Compile Include="Source\MySqlPacket.cs" />
<Compile Include="Source\MySqlScript.cs" />
Modified: trunk/MySql.Data/Provider/Source/ISSchemaProvider.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/ISSchemaProvider.cs 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/Source/ISSchemaProvider.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -231,14 +231,14 @@
string sql_mode = reader.GetString(1);
string body = reader.GetString(2);
- SqlTokenizer tokenizer = new SqlTokenizer(body);
+ MySqlTokenizer tokenizer = new MySqlTokenizer(body);
tokenizer.AnsiQuotes = sql_mode.IndexOf("ANSI_QUOTES") != -1;
tokenizer.BackslashEscapes = sql_mode.IndexOf("NO_BACKSLASH_ESCAPES") == -1;
string token = tokenizer.NextToken();
while (token != "(")
token = tokenizer.NextToken();
- int start = tokenizer.Index + 1;
+ int start = tokenizer.StartIndex + 1;
token = tokenizer.NextToken();
while (token != ")" || tokenizer.Quoted)
{
@@ -252,7 +252,7 @@
token = tokenizer.NextToken();
}
}
- return body.Substring(start, tokenizer.Index - start);
+ return body.Substring(start, tokenizer.StartIndex - start);
}
}
@@ -431,10 +431,10 @@
string sqlMode = row["SQL_MODE"].ToString();
int pos = 1;
- SqlTokenizer tokenizer = new SqlTokenizer(body);
+ MySqlTokenizer tokenizer = new MySqlTokenizer(body);
tokenizer.AnsiQuotes = sqlMode.IndexOf("ANSI_QUOTES") != -1;
tokenizer.BackslashEscapes = sqlMode.IndexOf("NO_BACKSLASH_ESCAPES") == -1;
-
+ tokenizer.ReturnComments = false;
string token = tokenizer.NextToken();
// this block will scan for the opening paren while also determining
@@ -466,6 +466,8 @@
parmRow["PARAMETER_MODE"] = mode;
token = tokenizer.NextToken();
}
+ if (tokenizer.Quoted)
+ token = token.Substring(1, token.Length - 2);
parmRow["PARAMETER_NAME"] = token;
// now parse data type
@@ -506,7 +508,7 @@
/// <summary>
/// Parses out the elements of a procedure parameter data type.
/// </summary>
- private string ParseDataType(DataRow row, SqlTokenizer tokenizer)
+ private string ParseDataType(DataRow row, MySqlTokenizer tokenizer)
{
StringBuilder dtd = new StringBuilder(
tokenizer.NextToken().ToUpper(CultureInfo.InvariantCulture));
@@ -514,9 +516,10 @@
string type = row["DATA_TYPE"].ToString();
string token = tokenizer.NextToken();
- if (tokenizer.IsSize)
+ if (token == "(")
{
- dtd.AppendFormat(CultureInfo.InvariantCulture, "({0})", token);
+ token = tokenizer.ReadParenthesis();
+ dtd.AppendFormat(CultureInfo.InvariantCulture, "{0}", token);
if (type != "ENUM" && type != "SET")
ParseDataTypeSize(row, token);
token = tokenizer.NextToken();
Modified: trunk/MySql.Data/Provider/Source/MySqlScript.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/MySqlScript.cs 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/Source/MySqlScript.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -244,7 +244,7 @@
int startPos = 0;
List<ScriptStatement> statements = new List<ScriptStatement>();
List<int> lineNumbers = BreakScriptIntoLines();
- SqlTokenizer tokenizer = new SqlTokenizer(query);
+ MySqlTokenizer tokenizer = new MySqlTokenizer(query);
tokenizer.AnsiQuotes = ansiQuotes;
tokenizer.BackslashEscapes = !noBackslashEscapes;
@@ -252,14 +252,14 @@
string token = tokenizer.NextToken();
while (token != null)
{
- if (!tokenizer.Quoted &&
- !tokenizer.IsSize)
+ if (!tokenizer.Quoted) // &&
+ //!tokenizer.IsSize)
{
int delimiterPos = token.IndexOf(Delimiter);
if (delimiterPos != -1)
{
- int endPos = tokenizer.Index - token.Length + delimiterPos;
- if (tokenizer.Index == query.Length-1)
+ int endPos = tokenizer.StopIndex - token.Length + delimiterPos;
+ if (tokenizer.StopIndex == query.Length-1)
endPos++;
string currentQuery = query.Substring(startPos, endPos-startPos);
ScriptStatement statement = new ScriptStatement();
@@ -274,7 +274,7 @@
}
// now clean up the last statement
- if (tokenizer.Index > startPos)
+ if (tokenizer.StartIndex > startPos)
{
string sqlLeftOver = query.Substring(startPos).Trim();
if (!String.IsNullOrEmpty(sqlLeftOver))
Modified: trunk/MySql.Data/Provider/Source/PreparableStatement.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/PreparableStatement.cs 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/Source/PreparableStatement.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -68,7 +68,7 @@
{
// strip out names from parameter markers
string text;
- ArrayList parameter_names = PrepareCommandText(out text);
+ List<string> parameter_names = PrepareCommandText(out text);
// ask our connection to send the prepare command
MySqlField[] paramList = null;
@@ -176,26 +176,24 @@
/// the parameterMap array list that includes all the paramter names in the
/// order they appeared in the SQL
/// </remarks>
- private ArrayList PrepareCommandText(out string stripped_sql)
+ private List<string> PrepareCommandText(out string stripped_sql)
{
StringBuilder newSQL = new StringBuilder();
- ArrayList parameterMap = new ArrayList();
+ List<string> parameterMap = new List<string>();
- // tokenize the sql first
- ArrayList tokens = TokenizeSql(ResolvedCommandText);
- parameterMap.Clear();
-
- foreach (string token in tokens)
+ int startPos = 0;
+ string sql = ResolvedCommandText;
+ MySqlTokenizer tokenizer = new MySqlTokenizer(sql);
+ string parameter = tokenizer.NextParameter();
+ while (parameter != null)
{
- if (token[0] != '@' && token[0] != '?')
- newSQL.Append(token);
- else
- {
- parameterMap.Add(token);
- newSQL.Append("?");
- }
+ newSQL.Append(sql.Substring(startPos, tokenizer.StartIndex - startPos));
+ newSQL.Append("?");
+ parameterMap.Add(parameter);
+ startPos = tokenizer.StopIndex;
+ parameter = tokenizer.NextParameter();
}
-
+ newSQL.Append(sql.Substring(startPos));
stripped_sql = newSQL.ToString();
return parameterMap;
}
Modified: trunk/MySql.Data/Provider/Source/SchemaProvider.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/SchemaProvider.cs 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/Source/SchemaProvider.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -413,7 +413,7 @@
lowerBody = body.ToLower(CultureInfo.InvariantCulture);
}
- SqlTokenizer tokenizer = new SqlTokenizer(lowerBody);
+ MySqlTokenizer tokenizer = new MySqlTokenizer(lowerBody);
tokenizer.AnsiQuotes = sqlMode.IndexOf("ANSI_QUOTES") != -1;
tokenizer.BackslashEscapes = sqlMode.IndexOf("NO_BACKSLASH_ESCAPES") != -1;
@@ -430,7 +430,7 @@
}
private static void ParseConstraint(DataTable fkTable, DataRow table,
- SqlTokenizer tokenizer, bool includeColumns)
+ MySqlTokenizer tokenizer, bool includeColumns)
{
string name = tokenizer.NextToken();
DataRow row = fkTable.NewRow();
@@ -448,7 +448,7 @@
row["TABLE_SCHEMA"] = table["TABLE_SCHEMA"];
row["TABLE_NAME"] = table["TABLE_NAME"];
row["REFERENCED_TABLE_CATALOG"] = null;
- row["CONSTRAINT_NAME"] = name;
+ row["CONSTRAINT_NAME"] = name.Trim(new char[] { '\'', '`' });
ArrayList srcColumns = includeColumns ? ParseColumns(tokenizer) : null;
@@ -460,13 +460,13 @@
if (target2.StartsWith("."))
{
row["REFERENCED_TABLE_SCHEMA"] = target1;
- row["REFERENCED_TABLE_NAME"] = target2.Substring(1);
+ row["REFERENCED_TABLE_NAME"] = target2.Substring(1).Trim(new char[] { '\'', '`' });
tokenizer.NextToken(); // read off the '('
}
else
{
row["REFERENCED_TABLE_SCHEMA"] = table["TABLE_SCHEMA"];
- row["REFERENCED_TABLE_NAME"] = target1;
+ row["REFERENCED_TABLE_NAME"] = target1.Substring(1).Trim(new char[] { '\'', '`' }); ;
}
// if we are supposed to include columns, read the target columns
@@ -478,7 +478,7 @@
fkTable.Rows.Add(row);
}
- private static ArrayList ParseColumns(SqlTokenizer tokenizer)
+ private static ArrayList ParseColumns(MySqlTokenizer tokenizer)
{
ArrayList sc = new ArrayList();
string token = tokenizer.NextToken();
Modified: trunk/MySql.Data/Provider/Source/Statement.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/Statement.cs 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/Source/Statement.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -25,6 +25,7 @@
using MySql.Data.Common;
using System.Data;
using MySql.Data.MySqlClient.Properties;
+using System.Collections.Generic;
namespace MySql.Data.MySqlClient
{
@@ -149,9 +150,6 @@
private void InternalBindParameters(string sql, MySqlParameterCollection parameters,
MySqlPacket packet)
{
- // tokenize the sql
- ArrayList tokenArray = TokenizeSql(sql);
-
if (packet == null)
{
packet = new MySqlPacket(Driver.Encoding);
@@ -159,34 +157,21 @@
packet.WriteByte(0);
}
- // make sure our token array ends with a ;
- string lastToken = (string) tokenArray[tokenArray.Count - 1];
- if (lastToken != ";")
- tokenArray.Add(";");
-
- foreach (String token in tokenArray)
+ int startPos = 0;
+ MySqlTokenizer tokenizer = new MySqlTokenizer(sql);
+ tokenizer.ReturnComments = true;
+ string parameter = tokenizer.NextParameter();
+ while (parameter != null)
{
- if (token.Trim().Length == 0)
- continue;
- if (token == ";")
- {
- buffers.Add(packet);
- packet = new MySqlPacket(Driver.Encoding);
- packet.WriteByte(0);
- packet.Version = Driver.Version;
- continue;
- }
- if (token.Length >= 2 &&
- ((token[0] == '@' && token[1] != '@') ||
- token[0] == '?'))
- {
- if (SerializeParameter(parameters, packet, token))
- continue;
- }
-
- // our fall through case is to write the token to the byte stream
- packet.WriteStringNoNull(token);
+ packet.WriteStringNoNull(sql.Substring(startPos, tokenizer.StartIndex - startPos));
+ bool serialized = SerializeParameter(parameters, packet, parameter);
+ startPos = tokenizer.StopIndex;
+ if (!serialized)
+ startPos = tokenizer.StartIndex;
+ parameter = tokenizer.NextParameter();
}
+ packet.WriteStringNoNull(sql.Substring(startPos));
+ buffers.Add(packet);
}
/// <summary>
@@ -249,50 +234,5 @@
parameter.Serialize(packet, false);
return true;
}
-
- public ArrayList TokenizeSql(string sql)
- {
- bool batch = Connection.Settings.AllowBatch && Driver.SupportsBatch;
- char delim = Char.MinValue;
- bool escaped = false;
- ArrayList tokens = new ArrayList();
-
- sql = sql.Trim(';');
- int startIndex = 0;
- for (int i = 0; i < sql.Length; i++)
- {
- char c = sql[i];
- if (escaped)
- escaped = false;
- else if (c == delim)
- delim = Char.MinValue;
- else if (c == ';' && !escaped && delim == Char.MinValue && !batch)
- {
- tokens.Add(sql.Substring(startIndex, i - startIndex));
- tokens.Add(";");
- startIndex = i + 1;
- continue;
- }
- else if ((c == '\'' || c == '\"' || c == '`') && !escaped && delim == Char.MinValue)
- delim = c;
- else if (c == '\\')
- escaped = !escaped;
- else if ((c == '@' || c == '?') &&
- delim == Char.MinValue && !escaped)
- {
- tokens.Add(sql.Substring(startIndex, i - startIndex));
- startIndex = i;
- }
- else if (i > startIndex && (sql[startIndex] == '@' || sql[startIndex] == '?') &&
- !Char.IsLetterOrDigit(c) && c != '_' && c != '.' && c != '$')
- {
- tokens.Add(sql.Substring(startIndex, i - startIndex));
- startIndex = i;
- }
- }
- tokens.Add(sql.Substring(startIndex, sql.Length - startIndex));
- return tokens;
- }
-
}
}
Modified: trunk/MySql.Data/Provider/Source/command.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/command.cs 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/Source/command.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -753,13 +753,13 @@
{
MySqlCommand cmd = new MySqlCommand("SELECT @@sql_mode", Connection);
string sql_mode = cmd.ExecuteScalar().ToString().ToUpper(CultureInfo.InvariantCulture);
- SqlTokenizer tokenizer = new SqlTokenizer(CommandText);
+ MySqlTokenizer tokenizer = new MySqlTokenizer(CommandText);
tokenizer.AnsiQuotes = sql_mode.IndexOf("ANSI_QUOTES") != -1;
tokenizer.BackslashEscapes = sql_mode.IndexOf("NO_BACKSLASH_ESCAPES") == -1;
string token = tokenizer.NextToken().ToLower(CultureInfo.InvariantCulture);
while (token != null)
{
- if (token.ToLower(CultureInfo.InvariantCulture) == "values" &&
+ if (token.ToUpper(CultureInfo.InvariantCulture) == "VALUES" &&
!tokenizer.Quoted)
{
token = tokenizer.NextToken();
@@ -773,7 +773,7 @@
batchableCommandText += token;
token = tokenizer.NextToken();
if (token != null && (token == "," ||
- token.ToLower(CultureInfo.InvariantCulture) == "on"))
+ token.ToUpper(CultureInfo.InvariantCulture) == "ON"))
{
batchableCommandText = null;
break;
Added: trunk/MySql.Data/Provider/Source/common/MySqlTokenizer.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/common/MySqlTokenizer.cs (rev 0)
+++ trunk/MySql.Data/Provider/Source/common/MySqlTokenizer.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -0,0 +1,272 @@
+using System;
+using System.Text;
+using System.IO;
+using System.Collections.Generic;
+
+namespace MySql.Data.MySqlClient
+{
+ internal class MySqlTokenizer
+ {
+ private string sql;
+
+ private int startLine;
+ private int stopLine;
+ private int startIndex;
+ private int stopIndex;
+
+ private bool ansiQuotes;
+ private bool backslashEscapes;
+ private bool returnComments;
+ private bool multiLine;
+
+ private bool quoted;
+ private bool isComment;
+
+ private int pos;
+
+ public MySqlTokenizer(string input)
+ {
+ sql = input;
+ backslashEscapes = true;
+ multiLine = true;
+ pos = 0;
+ }
+
+ #region Properties
+
+ public bool AnsiQuotes
+ {
+ get { return ansiQuotes; }
+ set { ansiQuotes = value; }
+ }
+
+ public bool BackslashEscapes
+ {
+ get { return backslashEscapes; }
+ set { backslashEscapes = value; }
+ }
+
+ public bool MultiLine
+ {
+ get { return multiLine; }
+ set { multiLine = value; }
+ }
+
+/* public bool IsSize
+ {
+ get { return isSize; }
+ }*/
+
+ public bool Quoted
+ {
+ get { return quoted; }
+ private set { quoted = value; }
+ }
+
+ public bool IsComment
+ {
+ get { return isComment; }
+ }
+
+ public int StartIndex
+ {
+ get { return startIndex; }
+ }
+
+ public int StopIndex
+ {
+ get { return stopIndex; }
+ }
+
+ public int StartLine
+ {
+ get { return startLine; }
+ }
+
+ public int StopLine
+ {
+ get { return stopLine; }
+ }
+
+ public bool ReturnComments
+ {
+ get { return returnComments; }
+ set { returnComments = value; }
+ }
+
+ #endregion
+
+ public List<string> GetAllTokens()
+ {
+ List<string> tokens = new List<string>();
+ string token = NextToken();
+ while (token != null)
+ {
+ tokens.Add(token);
+ token = NextToken();
+ }
+ return tokens;
+ }
+
+ public string NextToken()
+ {
+ while (FindToken())
+ {
+ string token = sql.Substring(startIndex, stopIndex - startIndex).Trim();
+ return token;
+ }
+ return null;
+ }
+
+ public string NextParameter()
+ {
+ while (FindToken())
+ {
+ if ((stopIndex - startIndex) < 2) continue;
+ char c1 = sql[startIndex];
+ char c2 = sql[startIndex+1];
+ if (c1 != '@' && c1 != '?') continue;
+ if (c1 == '@' && c2 == '@') continue;
+ return sql.Substring(startIndex, stopIndex - startIndex);
+ }
+ return null;
+ }
+
+ public bool FindToken()
+ {
+ isComment = quoted = false; // reset our flags
+ startIndex = stopIndex = -1;
+
+ while (pos < sql.Length)
+ {
+ char c = sql[pos++];
+ if (Char.IsWhiteSpace(c)) continue;
+
+ if (c == '`' || c == '\'' || (c == '"' && AnsiQuotes))
+ ReadQuotedToken(c);
+ else if (c == '#' || c == '-' || c == '/')
+ AttemptToReadComment(c);
+ else
+ ReadUnquotedToken();
+ if (startIndex != -1) return true;
+ }
+ return false;
+ }
+
+ public string ReadParenthesis()
+ {
+ StringBuilder sb = new StringBuilder("(");
+ int start = StartIndex;
+ string token = NextToken();
+ while (true)
+ {
+ if (token == null)
+ throw new InvalidOperationException("Unable to parse SQL");
+ sb.Append(token);
+ if (token == ")" && !Quoted) break;
+ token = NextToken();
+ }
+ return sb.ToString();
+ }
+
+ private void AttemptToReadComment(char c)
+ {
+ // make sure the comment starts correctly
+ if (c == '/' && (pos >= sql.Length || sql[pos] != '*')) return;
+ if (c == '-' && ((pos + 1) >= sql.Length || sql[pos] != '-' || sql[pos + 1] != ' ')) return;
+
+ string endingPattern = "\n";
+ if (sql[pos] == '*')
+ endingPattern = "*/";
+
+ int startingIndex = pos-1;
+
+ int index = sql.IndexOf(endingPattern, pos);
+ if (index == -1)
+ index = sql.Length - 1;
+ else
+ index += endingPattern.Length;
+
+ pos = index;
+ if (ReturnComments)
+ {
+ startIndex = startingIndex;
+ stopIndex = index;
+ isComment = true;
+ }
+ }
+
+ private void CalculatePosition(int start, int stop)
+ {
+ startIndex = start;
+ stopIndex = stop;
+ if (!MultiLine) return;
+ }
+
+ private void ReadUnquotedToken()
+ {
+ startIndex = pos-1;
+
+ if (!IsSpecialCharacter(sql[startIndex]))
+ {
+ while (pos < sql.Length)
+ {
+ char c = sql[pos];
+ if (Char.IsWhiteSpace(c)) break;
+ if (IsSpecialCharacter(c)) break;
+ if (IsParameterMarker(c))
+ {
+ if (c != '@' || pos > startIndex + 1 || sql[startIndex] != '@') break;
+ }
+ pos++;
+ }
+ }
+
+ Quoted = false;
+ stopIndex = pos;
+ }
+
+ /// <summary>
+ /// Read a single quoted identifier from the stream
+ /// </summary>
+ /// <param name="quoteChar"></param>
+ /// <returns></returns>
+ private void ReadQuotedToken(char quoteChar)
+ {
+ startIndex = pos-1;
+ bool escaped = false;
+
+ while (pos < sql.Length)
+ {
+ char c = sql[pos];
+
+ if (c == quoteChar && !escaped)
+ break;
+
+ if (escaped)
+ escaped = false;
+ if (c == '\\' && BackslashEscapes)
+ escaped = true;
+ pos++;
+ }
+ pos++;
+ Quoted = true;
+ stopIndex = pos;
+ }
+
+ private bool IsQuoteChar(char c)
+ {
+ return c == '`' || c == '\'' || (c == '\"' && AnsiQuotes);
+ }
+
+ private bool IsParameterMarker(char c)
+ {
+ return c == '@' || c == '?';
+ }
+
+ private bool IsSpecialCharacter(char c)
+ {
+ return c == '(' || c == ')' || c == ',' || c == ';' || c == '-' || c == '/' || c == '#';
+ }
+ }
+}
Deleted: trunk/MySql.Data/Provider/Source/common/SqlTokenizer.cs
===================================================================
--- trunk/MySql.Data/Provider/Source/common/SqlTokenizer.cs 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Provider/Source/common/SqlTokenizer.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -1,141 +0,0 @@
-using System;
-using System.Text;
-
-namespace MySql.Data.Common
-{
- internal class SqlTokenizer
- {
- private string input;
- private int index;
- //private StringBuilder current;
- private bool ansiQuotes;
- private bool backslashEscapes;
- private bool inSize;
- private bool isSize;
- private bool quoted;
- private bool inParamters;
-
- public SqlTokenizer(string input)
- {
- this.input = input;
- index = -1;
- backslashEscapes = true;
- //current = new StringBuilder();
- }
-
- #region Properties
-
- public bool AnsiQuotes
- {
- get { return ansiQuotes; }
- set { ansiQuotes = value; }
- }
-
- public bool BackslashEscapes
- {
- get { return backslashEscapes; }
- set { backslashEscapes = value; }
- }
-
- public bool IsSize
- {
- get { return isSize; }
- }
-
- public bool Quoted
- {
- get { return quoted; }
- }
-
- public int Index
- {
- get { return index; }
- }
-
- #endregion
-
- public string NextToken()
- {
- char lastChar = Char.MinValue;
- bool escaped = false;
- char quoteChar = Char.MinValue;
- StringBuilder current = new StringBuilder();
- bool inComment = false;
- bool inLineComment = false;
- quoted = isSize = false;
-
- while ((index+1) < input.Length)
- {
- char c = input[++index];
-
- if (escaped)
- {
- current.Append(c);
- escaped = false;
- }
- else if (c == quoteChar)
- {
- quoted = true;
- return current.ToString();
- }
- else if (quoteChar != Char.MinValue)
- current.Append(c);
- else if ((c == '`' || c == '\'' || (c == '\"' && AnsiQuotes)) && !inSize)
- quoteChar = c;
- else if (c == '/' && lastChar == '*' && inComment)
- inComment = false;
- else if (c == '*' && lastChar == '/')
- {
- inComment = true;
- current.Remove(current.Length - 1, 1);
- }
- else if (inComment || inLineComment)
- {
- if (inLineComment && c == '\n')
- inLineComment = false;
- }
- else if (c == '\\' && BackslashEscapes)
- {
- escaped = true;
- current.Append(c);
- }
- else if (c == '#')
- inLineComment = true;
- else if (c == '-' && lastChar == '-')
- {
- current.Remove(current.Length - 1, 1);
- inLineComment = true;
- }
- else if ((c == ',' || c == ')' || c == '(') && current.Length == 0)
- {
- inParamters = true;
- return c.ToString();
- }
- else if (Char.IsWhiteSpace(c) || c == '(' || c == ')' ||
- (c == ',' && !inSize))
- {
- if (c == ',' || (c == ')' && !inSize))
- index--;
- if (c == ')' && inSize)
- {
- isSize = true;
- inSize = false;
- }
- if (c == '(' && inParamters)
- inSize = true;
- if (current.Length > 0)
- return current.ToString();
- }
- else
- current.Append(c);
- lastChar = c;
- }
-
- if (current.Length > 0)
- return current.ToString();
-
- return null;
- }
-
- }
-}
Modified: trunk/MySql.Data/Tests/MySql.Data.Tests.csproj
===================================================================
--- trunk/MySql.Data/Tests/MySql.Data.Tests.csproj 2008-08-15 20:05:27 UTC (rev 1382)
+++ trunk/MySql.Data/Tests/MySql.Data.Tests.csproj 2008-08-18 21:41:06 UTC (rev 1383)
@@ -2,7 +2,7 @@
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
- <ProductVersion>9.0.21022</ProductVersion>
+ <ProductVersion>9.0.30729</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{F29E5B3D-7F76-4CF9-BF5E-8E3A1377B1E4}</ProjectGuid>
<OutputType>Library</OutputType>
@@ -57,7 +57,9 @@
<Compile Include="Source\ConnectionStringBuilder.cs" />
<Compile Include="Source\ConnectionTests.cs" />
<Compile Include="Source\BulkLoading.cs" />
- <Compile Include="Source\ScriptExecution.cs" />
+ <None Include="Source\ScriptExecution.cs" />
+ <Compile Include="Source\SqlTokenizer.cs" />
+ <Compile Include="Source\Tokenizer.cs" />
<Compile Include="Source\CultureTests.cs" />
<Compile Include="Source\DataAdapterTests.cs" />
<Compile Include="Source\DataReaderTests.cs" />
Added: trunk/MySql.Data/Tests/Source/SqlTokenizer.cs
===================================================================
--- trunk/MySql.Data/Tests/Source/SqlTokenizer.cs (rev 0)
+++ trunk/MySql.Data/Tests/Source/SqlTokenizer.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -0,0 +1,58 @@
+using System.Reflection;
+using System.Collections.Generic;
+
+namespace MySql.Data.MySqlClient.Tests
+{
+ class SqlTokenizer
+ {
+ object tokenizer;
+
+ public SqlTokenizer(string sql)
+ {
+ tokenizer = typeof(MySqlConnection).Assembly.CreateInstance("MySql.Data.MySqlClient.MySqlTokenizer",
+ false, System.Reflection.BindingFlags.CreateInstance, null,
+ new object[] { sql }, null, null);
+ }
+
+ public bool ReturnComments
+ {
+ set
+ {
+ PropertyInfo pi = tokenizer.GetType().GetProperty("ReturnComments");
+ pi.SetValue(tokenizer, value, null);
+ }
+ }
+
+ public bool AnsiQuotes
+ {
+ set
+ {
+ PropertyInfo pi = tokenizer.GetType().GetProperty("AnsiQuotes");
+ pi.SetValue(tokenizer, value, null);
+ }
+ }
+
+ public bool Quoted
+ {
+ get
+ {
+ PropertyInfo pi = tokenizer.GetType().GetProperty("Quoted");
+ return (bool)pi.GetValue(tokenizer, null);
+ }
+ }
+
+ public string NextToken()
+ {
+ return (string)tokenizer.GetType().InvokeMember("NextToken",
+ System.Reflection.BindingFlags.InvokeMethod,
+ null, tokenizer, null);
+ }
+
+ public string NextParameter()
+ {
+ return (string)tokenizer.GetType().InvokeMember("NextParameter",
+ System.Reflection.BindingFlags.InvokeMethod,
+ null, tokenizer, null);
+ }
+ }
+}
Added: trunk/MySql.Data/Tests/Source/Tokenizer.cs
===================================================================
--- trunk/MySql.Data/Tests/Source/Tokenizer.cs (rev 0)
+++ trunk/MySql.Data/Tests/Source/Tokenizer.cs 2008-08-18 21:41:06 UTC (rev 1383)
@@ -0,0 +1,238 @@
+// Copyright (C) 2004-2007 MySQL AB
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as published by
+// the Free Software Foundation
+//
+// There are special exceptions to the terms and conditions of the GPL
+// as it is applied to this software. View the full text of the
+// exception in file EXCEPTIONS in the directory of this software
+// distribution.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+using System;
+using System.Data;
+using System.IO;
+using NUnit.Framework;
+using System.Collections;
+using System.Collections.Generic;
+using System.Reflection;
+
+namespace MySql.Data.MySqlClient.Tests
+{
+ [TestFixture]
+ public class Tokenizer : BaseTest
+ {
+
+#if !CF
+ [Test]
+ public void Simple()
+ {
+ SqlTokenizer tokenizer = new SqlTokenizer("SELECT * FROM Test");
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void DashSingleLineComment()
+ {
+ string comment = "-- this is my comment\r\n";
+ string sql = String.Format("SELECT {0} * FROM Test", comment);
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = true;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual(comment.Trim(), tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+
+ tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = false;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void HashSingleLineComment()
+ {
+ string comment = "#this is my comment\r\n";
+ string sql = String.Format("SELECT {0} * FROM Test", comment);
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = true;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual(comment.Trim(), tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+
+ tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = false;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void MultiLineComment()
+ {
+ string comment = "/* this is my comment \r\n lines 2 \r\n line 3*/";
+ string sql = String.Format("SELECT{0} * FROM Test", comment);
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = true;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual(comment.Trim(), tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+
+ tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = false;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void Parameter()
+ {
+ string sql = "SELECT * FROM Test WHERE id=@id AND id2=?id2";
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = true;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.AreEqual("WHERE", tokenizer.NextToken());
+ Assert.AreEqual("id=", tokenizer.NextToken());
+ Assert.AreEqual("@id", tokenizer.NextToken());
+ Assert.AreEqual("AND", tokenizer.NextToken());
+ Assert.AreEqual("id2=", tokenizer.NextToken());
+ Assert.AreEqual("?id2", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void NextParameter()
+ {
+ string sql = "SELECT * FROM Test WHERE id=@id AND id2=?id2";
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = true;
+ Assert.AreEqual("@id", tokenizer.NextParameter());
+ Assert.AreEqual("?id2", tokenizer.NextParameter());
+ Assert.IsNull(tokenizer.NextParameter());
+ }
+
+ [Test]
+ public void ParameterWithSpecialCharacters()
+ {
+ string sql = "SELECT * FROM Test WHERE id=@id_$123";
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = true;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("*", tokenizer.NextToken());
+ Assert.AreEqual("FROM", tokenizer.NextToken());
+ Assert.AreEqual("Test", tokenizer.NextToken());
+ Assert.AreEqual("WHERE", tokenizer.NextToken());
+ Assert.AreEqual("id=", tokenizer.NextToken());
+ Assert.AreEqual("@id_$123", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void StringLiteral()
+ {
+ string sql = "SELECT 'a', 1, 'b'";
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = false;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("'a'", tokenizer.NextToken());
+ Assert.AreEqual(",", tokenizer.NextToken());
+ Assert.AreEqual("1", tokenizer.NextToken());
+ Assert.AreEqual(",", tokenizer.NextToken());
+ Assert.AreEqual("'b'", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void UserVariable()
+ {
+ string sql = "SELECT 'a', 1, @@myVar";
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.ReturnComments = false;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("'a'", tokenizer.NextToken());
+ Assert.AreEqual(",", tokenizer.NextToken());
+ Assert.AreEqual("1", tokenizer.NextToken());
+ Assert.AreEqual(",", tokenizer.NextToken());
+ Assert.AreEqual("@@myVar", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void AnsiQuotes()
+ {
+ string sql = "SELECT 'a', \"a\", `a`";
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.AnsiQuotes = false;
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("'a'", tokenizer.NextToken());
+ Assert.IsTrue(tokenizer.Quoted);
+ Assert.AreEqual(",", tokenizer.NextToken());
+ Assert.AreEqual("\"a\"", tokenizer.NextToken());
+ Assert.IsFalse(tokenizer.Quoted);
+ Assert.AreEqual(",", tokenizer.NextToken());
+ Assert.AreEqual("`a`", tokenizer.NextToken());
+ Assert.IsTrue(tokenizer.Quoted);
+ Assert.IsNull(tokenizer.NextToken());
+ }
+
+ [Test]
+ public void ParseProcBody()
+ {
+ string sql = "CREATE PROCEDURE spTest(testid INT, testname VARCHAR(20)) BEGIN SELECT 1; END";
+ SqlTokenizer tokenizer = new SqlTokenizer(sql);
+ tokenizer.AnsiQuotes = false;
+ Assert.AreEqual("CREATE", tokenizer.NextToken());
+ Assert.AreEqual("PROCEDURE", tokenizer.NextToken());
+ Assert.AreEqual("spTest", tokenizer.NextToken());
+ Assert.AreEqual("(", tokenizer.NextToken());
+ Assert.AreEqual("testid", tokenizer.NextToken());
+ Assert.AreEqual("INT", tokenizer.NextToken());
+ Assert.AreEqual(",", tokenizer.NextToken());
+ Assert.AreEqual("testname", tokenizer.NextToken());
+ Assert.AreEqual("VARCHAR", tokenizer.NextToken());
+ Assert.AreEqual("(", tokenizer.NextToken());
+ Assert.AreEqual("20", tokenizer.NextToken());
+ Assert.AreEqual(")", tokenizer.NextToken());
+ Assert.AreEqual(")", tokenizer.NextToken());
+ Assert.AreEqual("BEGIN", tokenizer.NextToken());
+ Assert.AreEqual("SELECT", tokenizer.NextToken());
+ Assert.AreEqual("1", tokenizer.NextToken());
+ Assert.AreEqual(";", tokenizer.NextToken());
+ Assert.AreEqual("END", tokenizer.NextToken());
+ Assert.IsNull(tokenizer.NextToken());
+ }
+#endif
+ }
+}
| Thread |
|---|
| • Connector/NET commit: r1383 - in trunk/MySql.Data: Provider Provider/Source Provider/Source/common Tests Tests/Source | rburnett | 18 Aug |
