Vertica使用心得

对于连接Vertica，使用Squirrel(6.0.0_vertica-jdk5-6.0.0-0.jar)来进行连接，也使用vsql(6.0.0_vertica-client-6.0.0-0.64.exe)进行连接。 

 

varchar(25) mean 25 bytes(octets)

vertica中是使用UTF8来存储，对于非英语字符，如汉字会占用多个字节，要合理计算字段的长度。
 

对C#开发，示例如下：

1) 单条语句

VerticaConnection conn = DBHelper.GetConnection();
IDbCommand cmdDel = conn.CreateCommand();
cmdDel.CommandType = CommandType.Text;
cmdDel.CommandText = "delete from dbo.TestTable1 where Id=:Id";
DBHelper.AddParameter(cmdDel, "Id", DbType.Int64, id);
cmdDel.ExecuteNonQuery();

 

2) 大批量操作

DataTable dt = new DataTable();
dt.Columns.Add("Id", typeof(int));
dt.Columns.Add("Name", typeof(string));
dt.Columns.Add("IsOK", typeof(string));
dt.Columns.Add("Score", typeof(double));
dt.Columns.Add("Score1", typeof(double));
dt.Columns.Add("Score2", typeof(double));

dt.Rows.Add(new object[] { 20000001, "test name", "Y", 191.123456789123456, 692.123456789123456, 993.123556789123456 });
dt.Rows.Add(new object[] { 20000002, "test name2", "Y", 194.123456789123456, 695.123456789123456, 996.123656789123456 });
dt.Rows.Add(new object[] { 20000003, "test name3", "N", 1910.123456789123456, 920.123456789123456, 930.123956789123456 });

DBHelper.BulkCopy(dt, "dbo.TestBulk");

    public class DBHelper
    {
        //public static void AddParameter(VerticaCommand cmd, string parameterName, VerticaType parameterType, object parameterVal)
        //{
        //    DBHelper.AddParameter(cmd, new VerticaParameter(parameterName, parameterType, parameterVal));
        //}

        public static void AddParameter(IDbCommand cmd, string parameterName, DbType parameterType, object parameterVal)
        {
            IDbDataParameter param = cmd.CreateParameter();
            param.ParameterName = parameterName;
            param.DbType = parameterType;
            param.Value = parameterVal == null ? DBNull.Value : parameterVal;

            cmd.Parameters.Add(param);
        }

        private static VerticaConnection m_conn = null;
        public static VerticaConnection GetConnection()
        {
            if (m_conn == null)
            {
                VerticaConnection conn = new VerticaConnection(ConfigurationManager.AppSettings["VerticaConnection"]);
                m_conn = conn;
            }
            if (m_conn.State == ConnectionState.Closed)
            {
                m_conn.Open();
            }
            return m_conn;
        }

        public static void ClearConnection()
        {
            try
            {
                if (m_conn != null)
                {
                    m_conn.Close();
                    m_conn.Dispose();
                    m_conn = null;
                }
            }
            catch (Exception ex)
            {
                LogHelper.Log(ex);
            }
        }

        public static void BulkCopy(DataTable dt, string strTableName)
        {
            if (dt == null || dt.Rows.Count == 0)
            {
                return;
            }
            if (dt.Columns.Count == 0)
            {
                throw new Exception("The length of column cannot be zero.");
            }

            List<string> lstField = new List<string>();
            for (int colIndex = 0; colIndex < dt.Columns.Count; colIndex++)
            {
                lstField.Add(dt.Columns[colIndex].ColumnName);
            }
            string strFiledList = string.Format("({0})", string.Join(",", lstField.ToArray()));

            const char RowSplit = '\n';
            const char ColSplit = '\t';

            string strCopyStatement = string.Format("copy {0}{1} from stdin record terminator E'{2}' delimiter E'{3}' enforcelength no commit",
                strTableName, strFiledList, RowSplit, ColSplit);

            StringBuilder sbText = new StringBuilder();
            foreach (DataRow dr in dt.Rows)
            {
                bool bFirstField = true;
                for (int colIndex = 0; colIndex < dt.Columns.Count; colIndex++)
                {
                    string strVal = GetDataString(dr, colIndex);
                    if (bFirstField)
                    {
                        sbText.Append(strVal);
                        bFirstField = false;
                    }
                    else
                    {
                        sbText.AppendFormat("{0}{1}", ColSplit, strVal);
                    }
                }
                sbText.Append(RowSplit);
            }

            string strTemp = sbText.ToString();
            byte[] buff = Encoding.Default.GetBytes(strTemp);
            using (MemoryStream ms = new MemoryStream())
            {
                ms.Write(buff, 0, buff.Length);
                ms.Flush();
                ms.Position = 0;

                VerticaConnection conn = GetConnection();
                {
                    VerticaTransaction txn = conn.BeginTransaction();
                    Vertica.Data.VerticaClient.VerticaCopyStream vcs = new VerticaCopyStream(conn, strCopyStatement);

                    vcs.Start();
                    vcs.AddStream(ms, false);
                    vcs.Execute();

                    long insertedCount = vcs.Finish();

                    IList<long> lstRejected = vcs.Rejects;
                    if (lstRejected.Count > 0)
                    {
                        txn.Rollback();
                        conn.Close();

                        // Maybe need more detail info to show
                        throw new Exception("Bulk copy failure.");
                    }
                    else
                    {
                        txn.Commit();
                        conn.Close();
                    }
                }

                ms.Close();
            }
        }

        private static string GetDataString(DataRow dr, int colIndex)
        {
            string strVal = "";
            if (!dr.IsNull(colIndex))
            {
                // only consider int/string
                strVal = dr[colIndex].ToString();
            }
            return strVal;
        }

 

copy dbo.TestBulk(Id, Name, Message) from local 'd:\\test.txt' record terminator E'\n' delimiter E'\t'  enforcelength exceptions 'd:\\exception_20120807.log';

 

Use COPY with the FILLER keyword to skip the year, month, and day columns from the source file:

COPY t(year FILLER VARCHAR(10),
month FILLER VARCHAR(10),
day FILLER VARCHAR(10),
k AS TO_DATE(YEAR || MONTH || DAY, 'YYYYMMDD') )
FROM STDIN NO COMMIT

skip the AutoGenId column from the source file
COPY t(AutoGenId filler bigint,
EventValue )
FROM STDIN NO COMMIT

 

 

 

另外，Vertica中使用count时要注意，count会排除为NULL的记录。