历年沪深A股、香港H股票数据导入和实时数据更新展示

【项目需求】

把以CSV文件格式保存的历年历年沪深A股、香港H股票数据导入数据库,如沪深每日财务数据、沪深板块分类数据、沪深历年十大股东变迁数据、港股实时5分钟数据统计等,并且需要从供应商接口实时采集最新股票价格、成交信息并展示给投资者。

【项目疑难点】

  • 文件数据大、种类多,部分数据格式、日期的转换。
  • 实时数据更新要求响应快,采集程序需要高效可靠。


【代码举例】

StockImport.class.php  股票数据导入统一接口类

<?php defined('IN_HEAVEN') or die('Hacking Attempt!');
/**
 * 股票数据导入
 * 
 * PHP version 5
 * 
 * @category	4SWeb
 * @package     Admin
 * @subpackage  Custom
 * @version     SVN: $Id: StockImport.class.php 67 2014-11-25 15:12:29Z blin.z $
 */
import('COM.GZNC.Import.CSVImport');
import('ORG.Util.SQLKit');
import('ORG.Util.File');
import('ORG.Util.String');

class StockImport extends CSVImport{
	const ERR_GUESS_FILE = 1201; // 文件名不对
	
	/**
	 * 数据类型
	 * @var const
	 */
	// 沪深
	
	/**
	 * 沪深每日财务数据(2008年3月18日前,37项)
	 * @var const
	 */
	const SHSZ_FINANCE37 = 'SHSZ_finance37'; // 沪深每日财务数据(2008年3月18日前,37项)
	
	/**
	 * 沪深每日财务数据(2008年3月19日起,51项)
	 * @var const
	 */
	const SHSZ_FINANCE = 'SHSZ_finance'; // 沪深每日财务数据(2008年3月19日起,51项)
	
	/**
	 * 沪深板块分类数据
	 * @var const
	 */
	const SHSZ_BLOCK = 'SHSZ_block'; // 沪深板块分类数据
	
	/**
	 * 沪深历年财务报表明细数据_利润表
	 * @var const
	 */
	const SHSZ_INCOME_STATEMENT = 'SHSZ_income_statement'; // 沪深历年财务报表明细数据_利润表
	
	/**
	 * 沪深历年财务报表明细数据_现金流量表
	 * @var const
	 */
	const SHSZ_CASHFLOW_STATEMENT = 'SHSZ_cashflow_statement'; // 沪深历年财务报表明细数据_现金流量表
	
	/**
	 * 沪深历年财务报表明细数据_资产负债表
	 * @var const
	 */
	const SHSZ_BALANCE_SHEET = 'SHSZ_balance_sheet'; // 沪深历年财务报表明细数据_资产负债表
	
	/**
	 * 沪深历年财务报表明细数据_相关91项分析指标
	 * @var const
	 */
	const SHSZ_FINANCIAL_ANALYSIS_INDEX = 'SHSZ_financial_analysis_index'; // 沪深历年财务报表明细数据_相关91项分析指标
	
	/**
	 * 沪深历年股本变动数据
	 * @var const
	 */
	const SHSZ_CAPITAL_STOCK = 'SHSZ_capital_stock'; // 4-沪深历年股本变动数据
	
	/**
	 * 沪深历年十大股东变迁数据
	 * @var const
	 */
	const SHSZ_SHAREHOLDER = 'SHSZ_shareholder'; // 5-沪深历年十大股东变迁数据
	
	/**
	 * 沪深历年十大流通股东变迁数据
	 * @var const
	 */
	const SHSZ_TRADABLE_SHAREHOLDER = 'SHSZ_tradable_shareholder'; // 5-沪深历年十大流通股东变迁数据
	
	/**
	 * 沪深日线数据
	 * @var const
	 */
	const SHSZ_DAY = 'SHSZ_day'; // 6-沪深日线数据
	
	/**
	 * 沪深日线数据(向前复权)
	 * @var const
	 */
	const SHSZ_DAY_FORWARD = 'SHSZ_day_forward'; // 6-沪深日线数据 向前复权
	
	/**
	 * 沪深5分钟数据
	 * @var const
	 */
	const SHSZ_5MIN = 'SHSZ_5Min'; // 6-沪深5分钟数据
	
	/**
	 * 沪深5分钟数据(向前复权)
	 * @var const
	 */
	const SHSZ_5MIN_FORWARD = 'SHSZ_5Min_forward'; // 6-沪深5分钟数据 向前复权
	
	/**
	 * 沪深权息数据
	 * @var const
	 */
	const SHSZ_SPLIT = 'SHSZ_split'; // 6-沪深权息数据
	
	/**
	 * 沪深实时数据
	 * @var const
	 */
	const SHSZ_REALTIME = 'SHSZ_realtime'; // 6-沪深实时数据
	
	/**
	 * 沪深实时5分钟数据统计
	 * @var const
	 */
	const SHSZ_REALTIME_5MIN = 'SHSZ_realtime_5Min'; // 沪深实时5分钟数据统计
	
	// 港股
	
	/**
	 * 港股财务数据
	 * @var const
	 */
	const HK_FINANCE = 'HK_finance'; // 港股财务数据
	
	/**
	 * 港股行业数据
	 * @var const
	 */
	const HK_INDUSTRY = 'HK_industry'; // 港股行业数据
	
	/**
	 * 港股日线数据
	 * @var const
	 */
	const HK_DAY = 'HK_day'; // 港股日线数据
	
	/**
	 * 港股日线数据(向前复权)
	 * @var const
	 */
	const HK_DAY_FORWARD = 'HK_day_forward'; // 港股日线数据 向前复权
	
	/**
	 * 港股5分钟数据
	 * @var const
	 */
	const HK_5MIN = 'HK_5Min'; // 港股5分钟数据
	
	/**
	 * 港股5分钟数据 向前复权
	 * @var const
	 */
	const HK_5MIN_FORWARD = 'HK_5Min_forward'; // 港股5分钟数据 向前复权
	
	/**
	 * 港股权息数据
	 * @var const
	 */
	const HK_SPLIT = 'HK_split'; // 港股权息数据
	
	/**
	 * 港股实时数据
	 * @var const
	 */
	const HK_REALTIME = 'HK_realtime'; // 港股实时数据
	
	/**
	 * 港股实时5分钟数据统计
	 * @var const
	 */
	const HK_REALTIME_5MIN = 'HK_realtime_5Min'; // 港股实时5分钟数据统计
	
	/**
	 * 监听类型
	 */
	const LISTENER_IMPORT_RESULT = 1;
	
	/**
	 * 监听器
	 * @var array
	 */
	protected static $_listners = array();
	
	/**
	 * 统计
	 * @var array
	 */
	protected $_stats = array();
	
	/**
	 * 当前数据驱动
	 * @var string
	 */
	protected $_driver = NULL;
	
	/**
	 * SQL模式
	 * @var mixed
	 */
	protected $_sql_mode = '';
	
	/**
	 * 每日下载目录
	 * @param string
	 */
	protected $_down_path = './WStockDown/';
	
	/**
	 * 每日更新目录
	 * @param string
	 */
	protected $_update_path = './WStockUpdate/';
	
	/**
	 * 实时更新目录
	 * @param string
	 */
	protected $_realtime_path = './WStockRealtime/';
	
	/**
	 * 日志目录
	 * @param string
	 */
	protected $_log_path = './WStockLog/';
	
	/**
	 * 数据库日志id
	 */
	protected $_log_id = NULL;
	
	/**
	 * 是否写SQL语句日志
	 */
	protected $_log_sql = false;
	
	/**
	 * 是否为更新导入
	 * @param boolean
	 */
	protected $_is_update = false;
	
	/**
	 * 文件默认年份
	 * @var string
	 */
	protected $_default_file_year = NULL;
	
	/**
	 * 表格字段定义
	 * @var array
	 */
	protected static $_fields_maps = NULL;
	

	
	/**
	 * 最后一条SQL
	 * @var string
	 */
	protected $_last_sql = NULL;
	
	public function __construct($driver='stockimport'){
		$this->_driver = $driver;
		
		// 导入工作目录
		$stock_path = ($t=C('STOCK_WORK_PATH'))?$t:DATA_PATH;
		
		$this->_down_path = $stock_path . 'WStockDown/';
		$this->_update_path = $stock_path . 'WStockUpdate/';
		$this->_realtime_path = $stock_path . 'WStockRealtime/';
		$this->_log_path = $stock_path . 'WStockLog/';
		
		/*
		if(!file_exists($this->_update_path) && !mkdir($this->_update_path)){
			return $this->_trigger_error("Cann't create update path {$this->_update_path}", SCRIPT_ERR_CONFIG, $this->_update_path, __FILE__, __LINE__);
		}*/
		
		/*
		if(!file_exists($this->_log_path) && !mkdir($this->_log_path)){
			return $this->_trigger_error("Cann't create log path {$this->_log_path}", SCRIPT_ERR_CONFIG, $this->_log_path, __FILE__, __LINE__);
		}*/
		
		//$this->_log = $log_dir . '/' .  $this->_driver . '_' . date('Ymd') . '.log';
		
		$log_dir = $this->_get_log_dir(date('Y-m-d'), true);
		$this->_log = $log_dir .  $this->_driver. '_' . date('Ymd') . '.log';
		
		$this->_logLevel = ($l=C('STOCK_IMPORT_LOG_LEVEL'))?$l:Log::INFO;
		
		// 忽略空数据、文件不存在错误
		$this->_ignoreError = array(self::ERR_EMPTY_DATA, self::ERR_FILE_READ, self::ERR_HTTP_REQUEST, self::ERR_GUESS_FILE);
		
		// 是否把SQL语句写入日志?
		$this->_log_sql = C('STOCK_IMPORT_LOG_SQL');
	}
	
	/**
	 * 数据导入实例工厂
	 * @param string $driver 数据类型名称
	 * @return object
	 */
	public static function factory($driver){
		$class = ucfirst(preg_replace("/_([a-zA-Z0-9])/e", "strtoupper('\\1')", $driver));
		if(!class_exists($class)){
			require 'Drivers/' . $class . '.class.php';
		}
		
		if(!class_exists($class)){
			return self::throw_exception("Driver {$driver} not exists", SCRIPT_ERR_CONFIG, $driver, __FILE__, __LINE__);
		}

		$obj = new $class($driver);
		return $obj;
	}
	
	/**
	 * 数据导入单入口
	 * @param string $file_or_date 数据文件,或者每日更新日期(格式:2014-08-23)
	 * @param string $driver 数据类型,不传的话,自动根据数据文件名称判断数据类型
	 * @param string $consoleOutput 是否把日志输出到终端屏幕
	 * @param string $default_file_year 默认年份,有些数据文件按月份命名,需要传入默认的年份,组合成数据的日期
	 * @param mixed $run_specific_driver 只导入指定的类型的数据,部分压缩数据包,里面包括多个数据文件
	 * @return mixed 返回导入行数
	 */
	public static function import($file_or_date, $driver=NULL, $consoleOutput=true, $default_file_year=NULL, $run_specific_driver=NULL){
		// auto detect driver & date from file name
		if(!$driver){
			if(is_dir($file_or_date)){
				import('ORG.Util.File');
				
				$aFiles = File::getDirFiles($file_or_date, array('zip', 'rar'));
				
				sort($aFiles, SORT_NATURAL);

			}else{
				$aFiles = array($file_or_date);
			}
			
			foreach($aFiles as $file){
				$detect_info = self::guess_driver_from_file_name($file);
				if(!$detect_info){
					return self::throw_exception("Cann't detect driver for file {$file}", SCRIPT_ERR_CONFIG, $file, __FILE__, __LINE__);
				}
					
				$driver = $detect_info['driver'];
				$date = $detect_info['date'];
					
				if(is_array($driver)){
					foreach($driver as $d){
						if(!$run_specific_driver || $run_specific_driver==$d){
							self::import(array('date'=>$date, 'down'=>$file), $d, $consoleOutput, $default_file_year);
						}
					}
				}else{
					self::import(array('date'=>$date, 'down'=>$file), $driver, $consoleOutput, $default_file_year);
				}
			}
			
			return;
		}
		
		$obj = self::factory($driver);
		if(!$obj){
			return self::throw_exception("Driver {$driver} not exists", SCRIPT_ERR_CONFIG, $driver, __FILE__, __LINE__);
		}
		
		if(is_array($file_or_date) || self::filter_date($file_or_date)){
			$func = 'importUpdate';
		}elseif(is_file($file_or_date)){
			$func = 'importFile';
		}elseif(is_dir($file_or_date)){
			$func = 'importFolder';
		}else{
			return self::throw_exception("File or folder {$file_or_date} not exists", self::ERR_FILE_READ, $file_or_date, __FILE__, __LINE__);
		}
		
		$obj->consoleOutput($consoleOutput);
		if($default_file_year){
			$obj->default_file_year($default_file_year);
		}
		
		return $obj->$func($file_or_date);
	}
	
	/**
	 * 返回数据类型对应的字段定义
	 * @param string $driver 数据类型
	 * @param bool $isUpdate 是否为每日更新,历史数据和每日更新数据的表结构存在不一样的情况
	 * @return array
	 */
	public static function getFields($driver, $isUpdate=false){
		if(empty(self::$_fields_maps)){
			self::$_fields_maps = require APP_ROOT_PATH . 'app/Web/Inc/stock_fields.inc.php';
		}
		
		if($isUpdate && isset(self::$_fields_maps[$driver]["UPDATE"])){
			return self::$_fields_maps[$driver]["UPDATE"];
		}else{
			return isset(self::$_fields_maps[$driver]["IMPORT"])?self::$_fields_maps[$driver]["IMPORT"]:self::$_fields_maps[$driver];
		}
	}
	
	/**
	 * 返回数据类型对应表名
	 * @param string $driver 数据类型
	 * @return string
	 */
	public static function getTable($driver){
		static $_SHSZ_TABLES = array(
				self::SHSZ_5MIN => 'shsz_5min',
				self::SHSZ_5MIN_FORWARD => 'shsz_5min_forward',
				self::SHSZ_BALANCE_SHEET => 'shsz_balance_sheet',
				self::SHSZ_BLOCK => 'shsz_block',
				self::SHSZ_CAPITAL_STOCK => 'shsz_capital_stock',
				self::SHSZ_CASHFLOW_STATEMENT => 'shsz_cashflow_statement',
				self::SHSZ_DAY => 'shsz_day',
				self::SHSZ_DAY_FORWARD => 'shsz_day_forward',
				self::SHSZ_FINANCE => 'shsz_finance',
				self::SHSZ_FINANCE37 => 'shsz_finance37',
				self::SHSZ_FINANCIAL_ANALYSIS_INDEX => 'shsz_financial_analysis_index',
				self::SHSZ_INCOME_STATEMENT => 'shsz_income_statement',
				self::SHSZ_SHAREHOLDER => 'shsz_shareholder',
				self::SHSZ_SPLIT => 'shsz_split',
				self::SHSZ_TRADABLE_SHAREHOLDER => 'shsz_tradable_shareholder',
				
				self::HK_FINANCE => 'hk_finance',
				self::HK_INDUSTRY => 'hk_industry',
				self::HK_SPLIT => 'hk_split',
				self::HK_5MIN => 'hk_5min',
				self::HK_5MIN_FORWARD => 'hk_5min_forward',
				self::HK_DAY => 'hk_day',
				self::HK_DAY_FORWARD => 'hk_day_forward',

		);
		
		return $_SHSZ_TABLES[$driver];
	}
	
	/**
	 * 返回对应数据类型的归档天数
	 * @param string $driver
	 * @return number|false
	 */
	public static function getArchiveDays($driver){
		switch($driver){
			case self::SHSZ_DAY:
			case self::SHSZ_DAY_FORWARD:
			case self::HK_DAY:
			case self::HK_DAY_FORWARD:
				$day = C('STOCK_ARCHIVE_DAY');
				return $day;
				break;
				
			case self::SHSZ_5MIN:
			case self::SHSZ_5MIN_FORWARD:
			case self::HK_5MIN:
			case self::HK_5MIN_FORWARD:
				$day = C('STOCK_ARCHIVE_5MIN');
				return $day;
				break;
				
			default:
				return false;
		}
	}
	
	/**
	 * 导入每日更新
	 * @param string $date 日期,格式如:2014-08-23
	 * @param string $extension 每日更新数据文件后缀名
	 * @return boolean
	 */
	public function importUpdate($date, $extension='csv'){
		return $this->_importUpdate($date, $extension);
	}
	
	/**
	 * 导入文件夹内数据
	 * @param string $folder 文件夹
	 * @param string $extension 数据文件后缀名
	 */
	public function importFolder($folder, $extension='csv'){
		return $this->_importFolder($folder, $extension);
	}
	
	/**
	 * 设置默认年份
	 * @param string $year
	 */
	public function default_file_year($year=NULL){
		if(is_null($year)){
			return $this->_default_file_year;
		}else{
			$this->_default_file_year = $year;
		}
	}
	
	/**
	 * 设置SQL模式
	 * @param string $strict 是否为严格模式,可以传true/false/模式名称
	 */
	public function sql_mode($strict=true){
		if($strict){
			$this->_sql_mode = $strict===true?'TRADITIONAL':$strict;
		}else{
			$this->_sql_mode = '';
		}
		
		$this->_VM()->query('SET sql_mode = "' . $this->_sql_mode . '"');
	}

	/**
	 * 手机号码过滤
	 * @param string $v
	 * @return string
	 */
	public static function filter_mobile($v){
		if(preg_match('/0?(1[0-9]{10})/', $v, $m)){
			return $m[1];
		}else{
			return '';
		}
	}
	
	/**
	 * 单位万转换
	 * @param number $v
	 * @return number
	 */
	public static function filter_tenthousands($v){
		//return $v?$v*10000:0;
		return $v?bcmul($v, 10000, 0):0;
	}
	
	/**
	 * 单位前转换
	 * @param number $v
	 * @return number
	 */
	public static function filter_thousand($v){
		//return $v?$v*1000:0;
		return $v?bcmul($v, 1000, 0):0;
	}
	
	/**
	 * 转换为整型
	 * @param number $v
	 * @return number
	 */
	public static function filter_intval($v){
		return bcadd($v, 0, 0);
	}
	
	/**
	 * 日期转换
	 * @param string $v
	 * @return string
	 */
	public static function filter_date($v){
		if(!$v){
			return false;
		}

		if(preg_match('/^([0-9]{4})([0-9]{2})([0-9]{2})$/', $v, $m)
		|| preg_match('/^([0-9]{4})[\/|\-]{1}([0-9]{1,2})[\/|\-]{1}([0-9]{1,2})$/', $v, $m)
		|| preg_match('/^([0-9]{4})[\/|\-]{1}([0-9]{1,2})[\/|\-]{1}([0-9]{1,2})\s+[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$/', $v, $m)
		){
			return $m[1] . '-' . $m[2] . '-' . $m[3];
		}else{
			// 尝试 Converts a Gregorian date to Julian Day Count
			return  (is_numeric ($v) && ($v=self::filter_convert_datetime($v)))?$v:false;
		}
	}
	
	/**
	 * 日期时间转换
	 * @param string $v
	 * @return string
	 */
	public static function filter_datetime($v){
		if(preg_match('/^([0-9]{4})[\/|\-]{1}([0-9]{1,2})[\/|\-]{1}([0-9]{1,2})\s+([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})$/', $v, $m)
		){
			return $m[1] . '-' . $m[2] . '-' . $m[3] . ' ' . $m[4];
		}else{
			return false;
		}
	}
	
	public static function filter_stock_code($v){
		if(preg_match('/^(SH|SZ)[0-9]{6}$/i', $v) || preg_match('/^HK[0-9]{5}$/i', $v)){
			return strtoupper($v);
		}elseif(preg_match('/^HK([0-9]{4})$/i', $v, $m)){
			return strtoupper('HK0' . $m[1]);
		}else{
			return false;
		}
	}

	protected function _batch_exec_sqls($sql, $delimiter=';'){
		$oVirtualModel = $this->_VM();
	
		if(is_string($sql)){
			$aSQLs = SQLKit::parse_sqls($sql);
		}else{
			$aSQLs = $sql;
		}
	
		$total = 0;
		$succs = 0;
		$fails = 0;
	
		$this->_log("Start batch execute SQLs");
	
		foreach ($aSQLs as $sql){
			if(!$sql) continue;
			$this->_log("Query: " . $sql);
			$result = $oVirtualModel->execute($sql);
			$this->_log("Result is " . $result?'succ':'fail: ' . $oVirtualModel->getDbError());
	
			if($result){
				$this->_setError("导入失败:" . $oVirtualModel->getDbError());
			}
	
			$total++;
			if($result){
				$succs++;
			}else{
				$fails++;
			}
		}
	
		$this->_log("Finished!");
		$this->_log("Total executed: " . $total . ", succ: " . $succs . ", fail: " . $fails);
	}
	
	protected function _VM(){
		if(!$this->_oVirtualModel){
			import('COM.GZNC.VirtualModel');
			$this->_oVirtualModel = new VirtualModel();
		}
	
		return $this->_oVirtualModel;
	}
	
	/**
	 * 使用mysqli连接数据库
	 * @return object
	 */
	protected function _mysqli(){
		static $mysqli = NULL;
		
		if($mysqli){
			return $mysqli;
		}
		
		// 写入数据库
		$mysqli = new mysqli('p:' . C('DB_HOST'), C('DB_USER'), C('DB_PWD'), C('DB_NAME'), C('DB_PORT'));
		
		/*
		 * This is the "official" OO way to do it,
		* BUT $connect_error was broken until PHP 5.2.9 and 5.3.0.
		*/
		if ($mysqli->connect_error) {
			return $this->_trigger_error('Connect DB Error (' . $mysqli->connect_errno . ') '
					. $mysqli->connect_error
					, self::ERR_DB_CONNECT
					, NULL
					, __FILE__
					, __LINE__
			);
		}
		
		/* change character set to utf8 */
		if (!$mysqli->set_charset("utf8")) {
			return $this->_trigger_error(sprintf("Error loading character set utf8: %s\n", $mysqli->error)
					, self::ERR_DB_QUERY
					, NULL
					, __FILE__
					, __LINE__
			);
		}
		
		return $mysqli;
	}
	

	/**
	 * 保存行数据到数据库
	 * @param string $table 表明
	 * @param string|array $key 表主键
	 * @param string $file 数据文件名称
	 * @param int $line 当前行值,如1,2,3,4
	 * @param array $fieldValues 字段回调函数处理过后的列值
	 * @param array $rowValues 原始列植
	 * @param array $options 选项设置
	 * @param bool $replace 是否用替换方式写入数据数据库
	 * @return string
	 */
	public function _cb_row_save($table, $key, $file, $line, $fieldValues, $rowValues, $options=NULL, $replace=false){
		$oVirtualModel = $this->_VM();

		if(!$replace){
			$exist_row = false;
			if($key){
				$where = is_array($key)?$key:array($key => $fieldValues[$key]);
					
				//----------------------
				// 注意:
				// 2014-10-12 zhongyw
				// Cli db->escapeString() 内的mysql_escape_string方法导入大数据时存在内存泄漏情况
				// 改为自行生成查询SQL
				//$exist_row = $oVirtualModel->table($table)->where($where)->find();
				//--------------------------
				$exist_sql = "SELECT";
				if(is_array($key)){
					$exist_sql .= " " . implode(', ', array_keys($key));
				}else{
					$exist_sql .= " `" . $key . "`";
				}
			
				if(!empty($options['checkUpdateDate'])){
					$exist_sql .= ", update_date";
				}
					
				$exist_sql .= " FROM `" . $table . "`";
				$exist_sql .= " WHERE " . SQLKit::where_expr($where);
				$exist_sql .= " LIMIT 1";
					
				$exist_result = $oVirtualModel->query($exist_sql);
					
				$exist_row = !empty($exist_result) && is_array($exist_result)?$exist_result[0]:false;
			}
			
			if(empty($exist_row)){
				if(!empty($options['_addFieldValues']) && is_array($options['_addFieldValues'])){
					$fieldValues = array_merge($fieldValues, $options['_addFieldValues']);
				}
					
				$sql = SQLKit::insert_stmt($table, $fieldValues);
			
				$operation = "Insert";
			}else{
				if(!empty($options['_updateFieldValues']) && is_array($options['_updateFieldValues'])){
					$fieldValues = array_merge($fieldValues, $options['_updateFieldValues']);
				}
					
				if(!empty($options['checkUpdateDate'])
				&& !empty($exist_row['update_date']) && !empty($fieldValues['update_date'])
				&& str_replace(array('-', '/'), '', $exist_row['update_date']) > str_replace(array('-', '/'), '', $fieldValues['update_date'])
				){
					$this->_log("Exist record date(" . $exist_row['update_date'] . ") is newer than update line:" . $line . " date (" . $fieldValues['update_date'] . ")", LOG::WARN);
					$this->_log("Skip Update");
					return true;
				}
					
				$sql = SQLKit::update_stmt($table, $fieldValues, $where);
				$operation = "Update";
			}
		}else{
			if(!empty($options['_addFieldValues']) && is_array($options['_addFieldValues'])){
				$fieldValues = array_merge($fieldValues, $options['_addFieldValues']);
			}
			
			if(!empty($options['_updateFieldValues']) && is_array($options['_updateFieldValues'])){
				$fieldValues = array_merge($fieldValues, $options['_updateFieldValues']);
			}
				
			$sql = SQLKit::replace_stmt($table, $fieldValues);
				
			$operation = "Replace";
		}

		
		$this->_last_sql = $sql;
		
		if (! empty ( $options ['instantWrite'] ) && !empty($sql)) {
		
			$log_content = "Line {$line}" . ($this->_log_sql?": $sql":"");
			
			$dbError = NULL;
			
			try {
				$result = $oVirtualModel->execute ( $sql );
			} catch ( Exception $e ) {
				$result = false;
				$dbError = $e->getMessage ();
			}
			
			// $result可能=0,当update同样内容数据的时候,必须要通过判断===false
			if (false === $result) {
				if (! $dbError) {
					$dbError = $oVirtualModel->getDbError ();
				}
				
				$this->_log( $log_content . ": $sql");
				$this->_log($operation . " FAIL: $dbError" );
				
				return $this->_trigger_error ( "Line {$line} importing SQL execute Failed: " . $dbError
						, self::ERR_SQL_EXECUTE
						, array (
								'line' => $line,
								'sql' => $sql 
						)
						, __FILE__, __LINE__ 
				);
			} else {
				$this->_log ( $log_content . ": " . $operation . " SUCC" );
				return true;
			}
		} else {
			return $sql;
		}
	}
	
	
	/**
	 * 写入日志
	 * @param string $message 日志消息
	 * @param string $level 日志等级
	 * @return boolean
	 */
	protected function _log($message, $level=LOG::INFO){
		if($this->_log){
			if (is_callable ( $this->_log )) {
				static $aLogLevelMaps = array (
						LOG::EMERG => 0,
						LOG::ALERT => 1,
						LOG::CRIT => 2,
						LOG::ERR => 3,
						LOG::WARN => 4,
						LOG::NOTICE => 5,
						LOG::INFO => 6,
						LOG::DEBUG => 7 
				);
				
				if ($this->_log && $aLogLevelMaps [$level] > $aLogLevelMaps [$this->_logLevel]) {
					return false;
				}
				
				return Runder::run_callback ( $this->_log, array (
						$message,
						$level 
				) );
			}else{
				return parent::_log($message, $level, $this->_logLevel);
			}
		}else{
			return false;
		}
	}
	
	/**
	 * xls文件转为csv
	 * @param string $xls_file 源xls文件
	 * @param string $csv_file 目标csv文件
	 * @param string $overwrite 是否覆盖
	 * @return string 返回目标csv文件名称
	 */
	protected function _convertXls2Csv($xls_file, $csv_file=NULL, $overwrite=false){
		$this->_log("Try convert xls to csv: $xls_file ");
		
		if(!$csv_file){
			$csv_file = (!strcasecmp('xls', File::getExtension($xls_file))?substr($xls_file, 0, strrpos($xls_file, '.')):$xls_file) . '.csv';
		}
		
		if(!$overwrite && self::file_exists($csv_file)){
			$this->_log("CSV file already exists");
			$this->_log("Skip converting");
			return $csv_file;
		}
		
		$bin_maps = C('BIN_MAPS');
		$oCommand = new Command($bin_maps);

		
		$convert_xls_file = self::convert2filesystemname($xls_file);
		$to_csv_file = self::convert2filesystemname($csv_file);
         
		/*
        $this->_log("\$convert_xls_file: ". $convert_xls_file);
        $this->_log("\$to_csv_file: ". $to_csv_file);
        $this->_log("\$convert_xls_file exist ". (file_exists($convert_xls_file)?'Yes':'No'));
		*/

		if(!empty($bin_maps['libxls2csv'])){
			$cmd_status = $oCommand->libxls2csv($convert_xls_file, $to_csv_file, 'GBK', "", ",");
		}else{
			$cmd_status = $oCommand->xls2csv($convert_xls_file, $to_csv_file, 'cp936', 'cp936');
		}
		
		if(1==$cmd_status){
			return $this->_trigger_error("Cann't convert xls to csv file: " . $oCommand->cmdOutput("\n")
					, self::ERR_OTHER_MISC
					, array('driver'=>$this->_driver, 'xls_file' => $xls_file, 'csv_file'=> $csv_file)
					, __FILE__
					, __LINE__
			);
		}else{
			$this->_log($oCommand->cmdOutput("\n"));
			$this->_log("Convert Succ!");
			$this->_log("CSV file is " . $csv_file);
			return $csv_file;
		}
	}
	
	/**
	 * 解压每日下载文件
	 * @param string $down_file 下载文件
	 * @param string $update_file 解压目标文件
	 * @return boolean
	 */
	protected function _decompressDownFile($down_file, $update_file=null){
		$this->_log("Down file is {$down_file}");
		
		if(!self::file_exists($down_file)){
			return $this->_trigger_error("Down file not exists!"
					, self::ERR_FILE_READ
					, array('driver'=>$this->_driver, 'down_file' => $down_file)
					, __FILE__
					, __LINE__
			);
		}
		
		$down_fext = File::getExtension($down_file);
		switch(strtolower($down_fext)){
			case 'csv':
			case 'xls':
				// Copy to update folder
				$this->_log("Try copy to update folder as file: " . $update_file);

				if(!copy($down_file, $update_file)){
					return $this->_trigger_error("Cann't copy down file to update file!"
							, self::ERR_OTHER_MISC
							, array('driver'=>$this->_driver, 'down_file' => $down_file, 'update_file'=> $update_file)
							, __FILE__
							, __LINE__
					);
				}

				$this->_log("Copy Succ!");
				
				break;
			
			case 'zip':
			case 'rar':
				$this->_log("Try decompress down file");
				
				$update_dir = dirname($update_file);
				
				$oCommand = new Command(C('BIN_MAPS'));
				$cmd_status = $oCommand->decompress($down_file, $update_dir);
				if(1==$cmd_status){
					return $this->_trigger_error("Cann't decompress down file: " . $oCommand->cmdOutput("\n")
							, self::ERR_OTHER_MISC
							, array('driver'=>$this->_driver, 'down_file' => $down_file, 'update_file'=> $update_file)
							, __FILE__
							, __LINE__
					);
				}
				
				$this->_log($oCommand->cmdOutput("\n"));
				
				$this->_log("Decompress Succ!");
				$this->_log("Saved into update directory: {$update_dir}");
				
				break;
				
			default:
				return $this->_trigger_error("Unknown file type: " . $down_fext
						, self::ERR_OTHER_MISC
						, array('driver'=>$this->_driver, 'down_file' => $down_file, 'update_file'=> $update_file)
						, __FILE__
						, __LINE__
				);
		}
		
		return true;
	}
	

	/**
	 * 导入文件夹
	 * @param string $folder 文件夹,完整路径
	 * @param string $extension 文件扩展名
	 * @param string $namepattern 文件名匹配正则
	 */
	protected function _importFolder($folder, $extension='csv', $namepattern=NULL){
		import('ORG.Util.Debug');
	
		$debug_start_mark = 'start_import_folder';
		$debug_end_mark = 'end_import_folder';
	
		Debug::mark($debug_start_mark);
	
		$this->_log("+++++++++++++++++++++++++++++++++++++");
		$this->_log("START IMPORT FOLDER");
		$this->_log("Folder {$folder}");
	
		import('ORG.Util.File');
	
		$aFiles = File::getDirFiles($folder, $extension);
		
		sort($aFiles, SORT_NATURAL);

		foreach($aFiles as $file){
			if(!$namepattern || preg_match($namepattern, $file)){
				$this->importFile($file);
			}
		}
	
		Debug::mark($debug_end_mark);
	
		$performance = "Cost time " . Debug::useTime($debug_start_mark,$debug_end_mark).'s';
		$performance .= ", used memory " . Debug::useMemory($debug_start_mark,$debug_end_mark).'kb';
		$performance .= ", peak memory " . Debug::getMemPeak($debug_start_mark, $debug_end_mark).'kb';
		$this->_log( $performance );
	
		$this->_log("END IMPORT FOLDER.[$folder]");
		$this->_log("+++++++++++++++++++++++++++++++++++++");
	}
	
	/**
	 * 导入数据文件
	 * @param string $file 数据文件
	 * @param array $fields 数据字段定义
	 * @param array $options 选项
	 * @param string $file_content 数据内容,传的话不再从$file读取,实时数据更新时有用
	 * @throws Exception
	 * @return mixed
	 */
	protected function _importFile($file, $fields, $options=array(
			'titleRow' => 1,  //  标题行数
			'dataRow' => 2, // 数据开始行数
			'instantWrite' => true, // 实时写
			'rowCallback' => '_cb_row',
			'fieldCallback' => '_cb_field',
			
			'fromEncoding' => 'GBK',
			'toEncoding' => 'UTF-8',
	), $file_content=NULL){
		
		$this->_begin_dblog($file);
		
		import('ORG.Util.Debug');
	
		$debug_start_mark = 'start_import_file';
		$debug_end_mark = 'end_import_file';
	
		Debug::mark($debug_start_mark);
	
		$this->_log("----------------------------");
		$this->_log("START IMPORT FILE");
		$this->_log("File {$file}");
	
		$this->_stats['total'] = 0;
		$this->_stats['succ'] = 0;
		$this->_stats['fail'] = array();

		$this->sql_mode(true);
		try {
			$sqls = $this->_import ( self::convert2filesystemname ( $file ), $fields, $options, $file_content);
			
			// S4Web::debug_log($sqls);
			
			if (! $sqls) {
				$error = $this->getError ();
				
				$this->_log ( "Import Failed: " . $error, LOG::ERR );
				$this->_end_dblog ( false, $error );
				
				// 监听处理
				$this->_callListener(self::LISTENER_IMPORT_RESULT, array($file, $error, false));
				
				return false;
			}
		} catch ( Exception $e ) {
			$this->_end_dblog ( false, $e);
			
			// 监听处理
			$this->_callListener(self::LISTENER_IMPORT_RESULT, array($file, $e, false));
			
			throw  $e;
		}
	
		//$this->_batch_exec_sqls($sqls);
	
		$message = "Total found " . $this->_stats['total'] . ' records';
		$message .= ", Succ " . ($this->_stats['succ']?$this->_stats['succ']:"<B COLOR=RED>" . $this->_stats['succ'] . "</B>");
		$message .= ", Fail " . count($this->_stats['fail']);
		$message .= $this->_stats['fail']?" Lines: " . implode(',', $this->_stats['fail']):'';
		$message .= ".";
		//$this->_setMessage($message);
	
		$this->_log($message);
	
		Debug::mark($debug_end_mark);
		
		$this->_stats['costsecs'] = Debug::useTime($debug_start_mark,$debug_end_mark, 3);
		$this->_stats['usedmemory'] = str_replace(',', '', Debug::useMemory($debug_start_mark,$debug_end_mark));
		$this->_stats['peakmemory'] = str_replace(',', '', Debug::getMemPeak($debug_start_mark,$debug_end_mark));
	
		$performance = "Cost time " . Debug::useTime($debug_start_mark,$debug_end_mark).'s';
		$performance .= ", used memory " . Debug::useMemory($debug_start_mark,$debug_end_mark).'kb';
		$performance .= ", peak memory " . Debug::getMemPeak($debug_start_mark, $debug_end_mark).'kb';
		$this->_log( $performance );
	
		$this->_log("END IMPORT FILE.[$file]");
		$this->_log("----------------------------");

		$this->_end_dblog (true, $message . "\n" . $performance);
		
		// 监听处理
		$this->_callListener(self::LISTENER_IMPORT_RESULT, array($file, $message, $this->_stats));
	
		return $sqls;
	}
	
	/**
	 * 导入每日更新
	 * @param string $date 日期
	 * @param string $extension 文件后缀名
	 * @return boolean
	 */
	protected function _importUpdate($date, $extension='csv'){
		import('ORG.Util.Debug');
	
		$debug_start_mark = 'start_import_update';
		$debug_end_mark = 'end_import_update';
	
		Debug::mark($debug_start_mark);
	
		$this->_log("=======================================");
		$this->_log("START IMPORT UPDATE");
		
		$update_dir = $this->_get_update_dir($date, true);
		
		if(is_array($date)){
			$update_file = $date['update'];
			$down_file = $date['down'];
			$date = $date['date'];
		}elseif(self::filter_date($date)){
			$down_file = $this->_get_down_file($date);
			
			// 沪深财务四张表先解压再匹配名称
			switch ($this->_driver){
				case self::SHSZ_INCOME_STATEMENT:
				case self::SHSZ_BALANCE_SHEET:
				case self::SHSZ_CASHFLOW_STATEMENT:
				case self::SHSZ_FINANCIAL_ANALYSIS_INDEX:
					$this->_decompressDownFile($down_file, $update_dir . '/update_file');
					break;
			}
			
			$update_file = $this->_get_update_file($date);
			
		}elseif(is_file($date)){
			$down_file = $date;
			$guess_info = self::guess_driver_from_file_name($down_file);
			$date = $guess_info['date'];
		}else{
			$this->_log("Import failed: Unknown date or file for driver " . $this->_driver . ": {$date}");
			return false;
		}
		
		if(!$date || !self::filter_date($date)){
			$this->_log("Import failed: Unknown date for driver " . $this->_driver . ": {$date}");
			return false;
		}
		

		if(!$down_file){
			$down_file = $this->_get_down_file($date);
		}
		
		if(!$update_file){
			$update_file = $this->_get_update_file($date);
		}
		
		if(!$update_file){
			$this->_log("Import failed: Cann't get update file for driver " . $this->_driver . " on date {$date}");
			return false;
		}elseif(is_string($update_file)){
			$update_files = array($update_file);
		}else{
			$update_files = $update_file;
		}
		

		// set update importing
		$this->_is_update = true;
		$this->_update_date = $date;
		
		$this->_log("Date {$date}");
		$this->_log("Down File: " . $down_file);
		
		foreach($update_files as $update_file){
			$this->_log("Update File: " . $update_file);
			$this->_log("Check if Update file exists");
			
			$this->_update_file = $update_file;

			// Check update file existence, and Try to decompress down file
			if(!self::file_exists($update_file)){
				if((!$this->_decompressDownFile($down_file, $update_file) || !self::file_exists($update_file))){
					$this->_log("Import Failed: Update file not exists, Cann't decompress from down file!");
					return false;
				}
			}else{
				$this->_log("Update file already exists");
			}
			
			$update_fext = File::getExtension($update_file);
			if(!strcasecmp('xls', $update_fext) && !strcasecmp('csv', $extension) 
			&& (!($update_file = $this->_convertXls2Csv($update_file)) || !self::file_exists($update_file))){
				$this->_log("Import Failed: Update file not exists, Cann't convert file type from xls to csv!");
				return false;
			}
				
			$this->importFile($update_file);
		}
		
		Debug::mark($debug_end_mark);
	
		$performance = "Cost time " . Debug::useTime($debug_start_mark,$debug_end_mark).'s';
		$performance .= ", used memory " . Debug::useMemory($debug_start_mark,$debug_end_mark).'kb';
		$performance .= ", peak memory " . Debug::getMemPeak($debug_start_mark, $debug_end_mark).'kb';
		$this->_log( $performance );
	
		$this->_log("END IMPORT UPDATE.");
		$this->_log("=======================================");
		
		return true;
	}
	
	
	/**
	 * 根据数据类型和日期,返回自动下载文件名称
	 * 
	 * @param string $driver 数据类型
	 * @param string $date 日期
	 * @return false|string
	 */
	public static function guess_down_file_name($driver, $date){
		if(!is_numeric($date)){
			$timestamp = strtotime($date);
		}else{
			$timestamp = $date;
		}
		
		if(!$timestamp){
			return false;
		}
		
		$shortdate = date('md', $timestamp);
		$date = date('Ymd', $timestamp);
		
	
		
		switch ($driver) {
			case self::SHSZ_FINANCE :
				// wss0929fin.zip
				$name = 'wss' . $shortdate . 'fin.zip';
				break;
				
			case self::SHSZ_BLOCK :
				// wss0929blk.zip
				$name = 'wss' . $shortdate . 'blk.zip';
				break;
				
			case self::SHSZ_BALANCE_SHEET :
			case self::SHSZ_CASHFLOW_STATEMENT :
			case self::SHSZ_INCOME_STATEMENT :
			case self::SHSZ_FINANCIAL_ANALYSIS_INDEX :
				// ws0929rpt.zip
				$name = 'ws' . $shortdate . 'rpt.zip';
				break;
				
			case self::SHSZ_SHAREHOLDER :
			case self::SHSZ_TRADABLE_SHAREHOLDER:
				// wsSHSZ_Shareholders_20140929.rar
				$name = 'wsSHSZ_Shareholders_' . $date . '.rar';
				break;
				
			case self::SHSZ_CAPITAL_STOCK :
				// wsSHSZ_CapitalStocks_20140930_csv.zip
				$name = 'wsSHSZ_CapitalStocks_' . $date . '_csv.zip';
				break;

			/*	
			case self::SHSZ_DAY :
				// wss0929e.zip
				//$name = 'wss' . $shortdate . 'e.zip';
				break;
				
			case self::SHSZ_5MIN :
				// wss0929e5.zip
				$name = 'wss' . $shortdate . 'e5.zip';
				break;
			*/

			case self::SHSZ_SPLIT:
			case self::SHSZ_DAY:
			case self::SHSZ_DAY_FORWARD:
			case self::SHSZ_5MIN:
			case self::SHSZ_5MIN_FORWARD:
				// wstock_SHSZ_Day_5Min_SPLIT_20140930.zip
				$name = "wstock_SHSZ_Day_5Min_SPLIT_{$date}.zip";
				break;
				
			case self::HK_FINANCE :
				// wsHK_finance_20141114.zip
				$name = 'wsHK_finance_' . $date . '.zip';
				break;
				
			case self::HK_INDUSTRY :
				// wsHK_Industry_20141114.zip
				$name = 'wsHK_Industry_' . $date . '.zip';
				break;

			case self::HK_SPLIT:
			case self::HK_DAY:
			case self::HK_DAY_FORWARD:
			case self::HK_5MIN:
			case self::HK_5MIN_FORWARD:
				// wstock_HK_Day_5Min_SPLIT_20140930.zip
				$name = "wstock_HK_Day_5Min_SPLIT_{$date}.zip";
				break;

			default:
				return false;
		}
		
		return $name;
	}
	
	/**
	 * 根据数据类型和日期,返回自动下载解压后文件名称
	 *
	 * @param string $driver 数据类型
	 * @param string $date 日期
	 * @return false|string|array
	 */
	public static function guess_update_file_name($driver, $date, $updatedir=NULL){
		if(!is_numeric($date)){
			$timestamp = strtotime($date);
		}else{
			$timestamp = $date;
		}
		
		if(!$timestamp){
			return false;
		}
		
		$shortdate = date('md', $timestamp);
		$date = date('Ymd', $timestamp);
		
		switch ($driver) {
			case self::SHSZ_FINANCE :
				// wss0930finance.csv
				$name = 'wss' . $shortdate . 'finance.csv';
				break;
		
			case self::SHSZ_BLOCK :
				// wsSHSZ_Block_20140930.csv
				$name = 'wsSHSZ_Block_' . $date . '.csv';
				break;
		
			// @todo: 需要依据一定规则自动判断按季变化的文件名称
			case self::SHSZ_BALANCE_SHEET :
				// SHSZ_2013Q2-2014Q2_资产负债表.xls
				//$name = "SHSZ_2013Q3-2014Q3_资产负债表.xls";
				$namepattern = '^SHSZ_[0-9A-Z\-]+_资产负债表\.xls$';
				$name = self::_match_update_file_name($updatedir, $namepattern);
				break;
			case self::SHSZ_CASHFLOW_STATEMENT :
				// SHSZ_2013Q2-2014Q2_现金流量表.xls
				//$name = "SHSZ_2013Q3-2014Q3_现金流量表.xls";
				$namepattern = '^SHSZ_[0-9A-Z\-]+_现金流量表\.xls$';
				$name = self::_match_update_file_name($updatedir, $namepattern);
				break;
			case self::SHSZ_INCOME_STATEMENT :
				// SHSZ_2013Q2-2014Q2_利润表.xls
				//$name = "SHSZ_2013Q3-2014Q3_利润表.xls";
				$namepattern = '^SHSZ_[0-9A-Z\-]+_利润表\.xls$';
				$name = self::_match_update_file_name($updatedir, $namepattern);
				break;
			case self::SHSZ_FINANCIAL_ANALYSIS_INDEX :
				// SHSZ_2013Q2-2014Q2_相关91项分析指标.xls
				//$name = "SHSZ_2013Q3-2014Q3_相关91项分析指标.xls";
				$namepattern = '^SHSZ_[0-9A-Z\-]+_相关91项分析指标\.xls$';
				$name = self::_match_update_file_name($updatedir, $namepattern);
				break;
		
			case self::SHSZ_SHAREHOLDER :
				// SHSZ_Shareholders_2010Q3-20140929.xls
				$name = "SHSZ_Shareholders_2010Q3-{$date}.xls";
				break;
				
			case self::SHSZ_TRADABLE_SHAREHOLDER :
				// SHSZ_TradableShareholders_2010Q3-20140929.xls
				$name = "SHSZ_TradableShareholders_2010Q3-{$date}.xls";
				break;
		
			case self::SHSZ_CAPITAL_STOCK :
				// SZ_CapitalStocks_1991-20140929.csv
				// SH_CapitalStocks_1990-20140929.csv
				$name = array(
					"SZ_CapitalStocks_1991-{$date}.csv",
					"SH_CapitalStocks_1990-{$date}.csv",
				);
				break;
				
			/*
			 * case self::SHSZ_DAY : // wss20140929r.csv $name = "wss{$date}r.csv"; break; case self::SHSZ_5MIN : // wss0929f.csv $name = "wss{$shortdate}f.csv"; break;
			 */
				
			case self::SHSZ_SPLIT:
				// wstock_SHSZ_20141019_SPLITs.csv
				$name = "wstock_SHSZ_{$date}_SPLITs.csv";
				break;
				
			case self::SHSZ_DAY :
				// wstock_SHSZ_20140930_Day.csv
				$name = "wstock_SHSZ_{$date}_Day.csv";
				break;
			
			case self::SHSZ_DAY_FORWARD :
				// wstock_SHSZ_20140930_Day_Forward.csv
				$name = "wstock_SHSZ_{$date}_Day_Forward.csv";
				break;
			
			case self::SHSZ_5MIN :
				// wstock_SHSZ_20140930_5Min.csv
				$name = "wstock_SHSZ_{$date}_5Min.csv";
				break;
			
			case self::SHSZ_5MIN_FORWARD :
				// wstock_SHSZ_20140930_5Min_Forward.csv
				$name = "wstock_SHSZ_{$date}_5Min_Forward.csv";
				break;
				
			case self::HK_FINANCE :
				// wsHK_finance_20141114.csv
				$name = 'wsHK_finance_' . $date . '.csv';
				break;
				
			case self::HK_INDUSTRY :
				// wsHK_Industry_20141114.xls
				$name = 'wsHK_Industry_' . $date . '.xls';
				break;
				
			case self::HK_SPLIT:
				// wstock_HK_20141019_SPLITs.csv
				$name = "wstock_HK_{$date}_SPLITs.csv";
				break;
				
			case self::HK_DAY :
				// wstock_HK_20140930_Day.csv
				$name = "wstock_HK_{$date}_Day.csv";
				break;
				
			case self::HK_DAY_FORWARD :
				// wstock_HK_20140930_Day_Forward.csv
				$name = "wstock_HK_{$date}_Day_Forward.csv";
				break;
			
			case self::HK_5MIN :
				// wstock_HK_20140930_5Min.csv
				$name = "wstock_HK_{$date}_5Min.csv";
				break;
			
			case self::HK_5MIN_FORWARD :
				// wstock_HK_20140930_5Min_Forward.csv
				$name = "wstock_HK_{$date}_5Min_Forward.csv";
				break;
		
			default:
				return false;
		}
		
		return $name;
	}
	
	protected static function _match_update_file_name($updatedir, $namepattern){
		import('ORG.Util.File');
		$aFiles = File::getDirFiles($updatedir);
		
		import('ORG.Util.String');
		$aFiles = String::autoCharset($aFiles);
		
		// force set regular match encoding as UTF-8
		mb_regex_encoding('UTF-8');
		
		foreach($aFiles as $file){
			$filename = basename($file);
			if(mb_ereg($namepattern, $filename)){
				return $filename;
			}
		}
		
		return false;
	}
	
	/**
	 * 从文件名判断数据类型和日期
	 * 
	 * @param string $file 文件名
	 * @return false|array
	 */
	public static function guess_driver_from_file_name($file){

		$pathinfo = pathinfo($file);
		if(!$pathinfo || !($basename=$pathinfo['basename'])){
			return false;
		}
		
		// wss0929fin.zip
		if(preg_match('/^wss([0-9]{4})fin\.zip$/', $basename, $m)){
			$return = array(
					'driver' => self::SHSZ_FINANCE,
					'date' => date('Y') . $m[1],
			);
		}
		
		// wss0929blk.zip
		elseif(preg_match('/^wss([0-9]{4})blk\.zip$/', $basename, $m)){
			$return = array(
					'driver' => self::SHSZ_BLOCK,
					'date' => date('Y') . $m[1],
			);
		}
		
		// ws0929rpt.zip
		elseif(preg_match('/^ws([0-9]{4})rpt\.zip$/', $basename, $m)){
			$return = array(
					'driver' => array(
							self::SHSZ_BALANCE_SHEET, 
							self::SHSZ_CASHFLOW_STATEMENT, 
							self::SHSZ_INCOME_STATEMENT,
							self::SHSZ_FINANCIAL_ANALYSIS_INDEX
					),
					'date' => date('Y') . $m[1],
			);
		}
		
		// wsSHSZ_Shareholders_20140929.rar
		elseif(preg_match('/^wsSHSZ_Shareholders_([0-9]{8})\.rar$/', $basename, $m)){
			$return = array(
					'driver' => array(
							self::SHSZ_SHAREHOLDER,
							self::SHSZ_TRADABLE_SHAREHOLDER,
					),
					'date' => $m[1],
			);
		}
		
		// wsSHSZ_CapitalStocks_20140929_csv.zip
		elseif(preg_match('/^wsSHSZ_CapitalStocks_([0-9]{8})_csv\.zip$/', $basename, $m)){
			$return = array(
					'driver' =>self::SHSZ_CAPITAL_STOCK,
					'date' => $m[1],
			);
		}
		
		// wss0929e.zip
		elseif(preg_match('/^wss([0-9]{4})e\.zip$/', $basename, $m)){
			$return = array(
					'driver' => self::SHSZ_DAY,
					'date' => date('Y') . $m[1],
			);
		}
		
		// wss0929e5.zip
		elseif(preg_match('/^wss([0-9]{4})e5\.zip$/', $basename, $m)){
			$return = array(
					'driver' => self::SHSZ_5MIN,
					'date' => date('Y') . $m[1],
			);
		}
		
		// wstock_SHSZ_Day_5Min_20140930.zip
		elseif(preg_match('/^wstock_SHSZ_Day_5Min_([0-9]{8})\.zip$/', $basename, $m)){
			$return = array(
					'driver' => array(
							self::SHSZ_SPLIT,
							self::SHSZ_DAY,
							self::SHSZ_DAY_FORWARD,
							self::SHSZ_5MIN,
							self::SHSZ_5MIN_FORWARD,
					),
					'date' => $m[1],
			);
		}
		
		// wsHK_finance_20141114.zip
		elseif(preg_match('/^wsHK_finance_([0-9]{8})\.zip$/', $basename, $m)){
			$return = array(
					'driver' => self::HK_FINANCE,
					'date' => $m[1],
			);
		}
		
		// wsHK_Industry_20141114.zip
		elseif(preg_match('/^wsHK_Industry_([0-9]{8})\.zip$/', $basename, $m)){
			$return = array(
					'driver' => self::HK_INDUSTRY,
					'date' => $m[1],
			);
		}
		
		// wstock_HK_Day_5Min_20140930.zip
		elseif(preg_match('/^wstock_HK_Day_5Min_([0-9]{8})\.zip$/', $basename, $m)){
			$return = array(
					'driver' => array(
							self::HK_SPLIT,
							self::HK_DAY,
							self::HK_DAY_FORWARD,
							self::HK_5MIN,
							self::HK_5MIN_FORWARD,
					),
					'date' => $m[1],
			);
		}
		
		else{
			return false;
		}
		
		$return['date'] = self::filter_date($return['date']);

		if(!$return['date']){
			return false;
		}else{
			return $return;
		}
	}
	
	/**
	 * 转换为文件系统字符集编码的文件名称
	 * @param string $name
	 */
	public static function convert2filesystemname($file){
		return String::autoCharset($file, C('DEFAULT_CHARSET'), C('FILE_SYSTEM_ENCODING'));
	}
	
	/**
	 * 判断本地文件系统中文件是否存在
	 * @param string $file
	 * @return boolean
	 */
	public static function file_exists($file){
		$file = self::convert2filesystemname($file);
		return file_exists($file);
	}
	
	/**
	 * 返回每日更新解压后的文件名称
	 * @param string $date 日志
	 * @return string
	 */
	protected function _get_update_file($date){
		$update_dir = $this->_get_update_dir($date);
		$update_fname = self::guess_update_file_name($this->_driver, $date, $update_dir);
		if(!$update_fname){
			return $this->_trigger_error("Cann't guess update file name!"
					, self::ERR_GUESS_FILE
					, array('driver'=>$this->_driver, 'date' => $date)
					, __FILE__
					, __LINE__
			);
			
			return false;
		}

		if(is_array($update_fname)){
			$return = array();
			foreach($update_fname as $f){
				$return[] = $update_dir . $f;
			}
			return $return;
		}else{
			return  $update_dir . $update_fname;
		}
		
	}
	
	/**
	 * 返回更新(解压后)目录
	 * @param string $date 日期
	 * @param string $autocreate 目录不存在是否自动创建
	 * @return boolean|string
	 */
	protected function _get_update_dir($date, $autocreate=false){
		$update_folder = date('Ymd', strtotime($date));
		$update_folder .= '/';
		
		$update_dir = $this->_update_path . $update_folder;
		
		if($autocreate && !file_exists($this->_update_path) && !mkdir($this->_update_path)){
			return $this->_trigger_error("Cann't create update path!"
					, self::ERR_OTHER_MISC
					, array('driver'=>$this->_driver, 'date' => $date, 'update_path'=>$this->_update_path
					)
					, __FILE__
					, __LINE__
			);
		}
		
		if($autocreate && !file_exists($update_dir) && !mkdir($update_dir, 0777, true)){
			return $this->_trigger_error("Cann't create update dir!"
					, self::ERR_FILE_CREATE
					, array('driver'=>$this->_driver, 'date' => $date, 'update_dir'=>$update_dir
			)
					, __FILE__
					, __LINE__
			);
		}
		
		return $update_dir;
	}
	
	/**
	 * 返回每日下载文件
	 * @param string $date 日期
	 * @return boolean|string
	 */
	protected function _get_down_file($date){
		$down_fname = self::guess_down_file_name($this->_driver, $date);
		if(!$down_fname){
			return $this->_trigger_error("Cann't guest down file name!"
					, self::ERR_GUESS_FILE
					, array('driver'=>$this->_driver, 'date' => $date)
					, __FILE__
					, __LINE__
			);
		}
	
		return $this->_get_down_dir($date) . $down_fname;
	}
	
	/**
	 * 返回每日下载目录
	 * @param string $date 日期
	 * @return string
	 */
	protected function _get_down_dir($date){
		$down_folder = date('Ymd', strtotime($date));
		$down_folder .= '/';
	
		return $this->_down_path . $down_folder;
	}
	
	/**
	 * 返回实时更新数据目录
	 * @param string $date
	 * @param string $autocreate
	 * @return boolean|string
	 */
	protected function _get_realtime_dir($date, $autocreate=false){
		$folder = date('Ymd', strtotime($date));
		$folder .= '/';
	
		$realtime_dir = $this->_realtime_path . $folder;
	
		if($autocreate && !file_exists($this->_realtime_path) && !mkdir($this->_realtime_path)){
			return $this->_trigger_error("Cann't create realtime path!"
					, self::ERR_FILE_CREATE
					, array('driver'=>$this->_driver, 'date' => $date, 'realtime_path'=>$this->_realtime_path
					)
					, __FILE__
					, __LINE__
			);
		}
	
		if($autocreate && !file_exists($realtime_dir) && !mkdir($realtime_dir, 0777, true)){
			return $this->_trigger_error("Cann't create realtime dir!"
					, self::ERR_FILE_CREATE
					, array('driver'=>$this->_driver, 'date' => $date, 'realtime_dir'=>$realtime_dir
					)
					, __FILE__
					, __LINE__
			);
		}
	
		return $realtime_dir;
	}
	
	/**
	 * 返回日志目录
	 * @param string $date 日期
	 * @param string $autocreate 目录不存在是否自动创建
	 * @return boolean|string
	 */
	protected function _get_log_dir($date, $autocreate=false){
		$folder = date('Ymd', strtotime($date));
		$folder .= '/';
	
		$dir = $this->_log_path . $folder;
	
		if($autocreate && !file_exists($this->_log_path) && !mkdir($this->_log_path)){
			return $this->_trigger_error("Cann't create path!"
					, self::ERR_FILE_CREATE
					, array('driver'=>$this->_driver, 'date' => $date, 'path'=>$this->_log_path
					)
					, __FILE__
					, __LINE__
			);
		}
	
		if($autocreate && !file_exists($dir) && !mkdir($dir, 0777, true)){
			return $this->_trigger_error("Cann't create dir!"
					, self::ERR_FILE_CREATE
					, array('driver'=>$this->_driver, 'date' => $date, 'dir'=>$dir
					)
					, __FILE__
					, __LINE__
			);
		}
	
		return $dir;
	}
	
	/**
	 * 发送GET请求
	 * 
	 * @param string $url	链接
	 * @param string|array $data	参数
	 * @return string
	 */
	public function get($url, $data = null) {
		if(!$this->Http){
			import('ORG.Net.curl');
			$this->Http = new Curl();
		}
		
		if(!($return = $this->Http->get($url, $data)) && ($error=$this->Http->getError())){
			return $this->_trigger_error(
					  $error
					, self::ERR_HTTP_REQUEST
					, array('url' => $url, 'data' => $data, 'method' => 'get', 'response' => $return) 
					, __FILE__, __LINE__);
		}
		

		return $return;
	}
	
	protected function _begin_dblog($file){
		$this->_log_id = $this->_VM()->table('stock_update_log')
		->add(array(
				'begintime' => date('Y-m-d H:i:s'),
				'driver' => $this->_driver,
				'datafile' => $file,
				'logfile' => $this->_log,
				'updatestatus' => 'running',
		));
		
		return $this->_log_id;
	}
	
	protected function _end_dblog($status, $message=NULL){
		if(!$this->_log_id){
			return false;
		}
		
		return $this->_VM()->table('stock_update_log')
		->where(array('id'=>$this->_log_id))
		->save(array(
				'records' => $this->_stats['total'],
				'succs' => $this->_stats['succ'],
				'fails' => $this->_stats['fail'],
				
				'costsecs' => $this->_stats['costsecs'],
				'usedmemory' => $this->_stats['usedmemory'],
				'peakmemory' => $this->_stats['peakmemory'],
				
				'endtime' => date('Y-m-d H:i:s'),
				'updatestatus' => $status?'succ':'fail',
				'updatemsg' => (is_object($message) && $message instanceof Exception)?$message->__toString():$message,
		));
	}
	
	/**
	 * 从文件名分析出数据日期
	 * @param string $file 数据文件名称
	 * @param string $pattern 匹配正则
	 * @param string $default_year 默认年份
	 * @return string
	 */
	protected function _parse_file_date($file, $pattern, $default_year=NULL){
		static $_file_dates = array();
		
		$fidx = md5($file);
		
		// 分析文件日期
		//if(!$_file_dates[$fidx]){
		if(empty($_file_dates[$fidx])){
			$fileName = basename($file);
			$date = NULL;
			if(preg_match($pattern, $fileName, $m)){
				$date = $m[1];
				if(strlen($date)==4 && $default_year){
					$date = $default_year . $date;
				}
				$date = self::filter_date($date);
			}

			if(!$date){
				return $this->_trigger_error("Cann't parse date from file name"
						, self::ERR_OTHER_MISC
						, array('file' => $file)
						, __FILE__, __LINE__
				);
			}
			
			$_file_dates[$fidx] = $date;
		}
		
		return $_file_dates[$fidx];
	}

	/**
	 * 从数据文件名称分析出股票代码
	 * @param string $file 数据文件名称
	 * @param string $pattern 匹配正则
	 * @return string
	 */
	protected function _parse_file_stock_code($file, $pattern){
		static $_file_stocks = array();
		
		$fidx = md5($file);
		
		// 分析文件日期
		//if(!$_file_stocks[$fidx]){
		if(empty($_file_stocks[$fidx])){
			$fileName = basename($file);
			$date = NULL;
			if(preg_match($pattern, $fileName, $m)){
				$stock = $m[1];
				$stock = self::filter_stock_code($stock);
			}
			
			if(!$stock){
				return $this->_trigger_error("Cann't parse stock code from file name"
						, self::ERR_OTHER_MISC
						, array('file' => $file)
						, __FILE__, __LINE__
				);
			}
			
			$_file_stocks[$fidx] = $stock;
		}
		
		return $_file_stocks[$fidx];
	}
	
	/**
	 * 从数据文件名称分析出数据源
	 * @param string $file 数据文件
	 * @param number $limit 返回目录层次
	 * @return string
	 */
	protected function _parse_file_source($file, $limit=3){
		static $_file_sources = array();
		
		$fidx = md5($file);
		
		// 分析文件源
		if(empty($_file_sources[$fidx])){
			if (preg_match ( '~(((\/|\\\)[^\/\\\]+){1,' . $limit . '})$~', $file, $m )) {
				$source = substr($m[1], 1);
			}else{
				$source = $file;
			}
			$_file_sources[$fidx] = $source;
		}
		
		return $_file_sources[$fidx];
	}
	
	/**
	 * 更新沪深股票名称
	 * @param string $file 股票名称更新文件
	 * @return mixed
	 */
	public function updateSHSZStockName($file){
		$fields = array
			(
					'Symbol' => array
					(
							'column' => 'A',
							'title' => '代码'
					),
					'CName' => array
					(
							'column' => 'B',
							'title' => '中文'
					),
					'EName' => array
					(
							'column' => 'C',
							'title' => '英文简写'
					),
			);;
	
		$options = array(
				'titleRow' => 1,  //  标题行数
				'dataRow' => 2, // 数据开始行数
				'instantWrite' => true, // 实时写
				'fieldCallback' => array($this, '_cb_field_update_stock_name'),
				'rowCallback' => array($this, '_cb_row_update_stock_name'),
	
				'fromEncoding' => 'GBK',
				'toEncoding' => 'UTF-8',
	
				//'checkUpdateDate' => true, // 只更新比当前日期新的纪录
		);
	
		return $this->_importFile($file, $fields, $options);
	}
	
	/**
	 * 更新港股股票名称
	 * @param string $file 股票名称更新文件
	 * @return mixed
	 */
	public function updateHKStockName($file){
		$fields = array
		(
				'Symbol' => array
				(
						'column' => 'A',
						'title' => 'Code'
				),
				'EName' => array
				(
						'column' => 'B',
						'title' => 'English Name'
				),
		);;
	
		$options = array(
				'titleRow' => 1,  //  标题行数
				'dataRow' => 2, // 数据开始行数
				'instantWrite' => true, // 实时写
				'fieldCallback' => array($this, '_cb_field_update_stock_name'),
				'rowCallback' => array($this, '_cb_row_update_stock_name'),
	
				'fromEncoding' => 'GBK',
				'toEncoding' => 'UTF-8',
	
				//'checkUpdateDate' => true, // 只更新比当前日期新的纪录
		);
	
		return $this->_importFile($file, $fields, $options);
	}
	
	public function _cb_field_update_stock_name($file, $line, $column, $field, $value, $options=NULL){
		$value = self::filter_trim($value);
		switch($field){
			// 股票代码
			case 'stock':
				$value = self::filter_stock_code($value);
				if(!$value){
					return true;
				}
				break;
		}
	
		return $value;
	}
	
	public function _cb_row_update_stock_name($file, $line, $fieldValues, $rowValues, $options=NULL){
		$this->_stats['total']++;
	
		$table = 'stock_symbol';
		$key = 'Symbol';
	
		// 添加字段
		$options['_addFieldValues'] = array(
				'Name' => $fieldValues['CName']?$fieldValues['CName']:$fieldValues['EName'],
				'created_time' => date('Y-m-d H:i:s'),
		);
		
		// 更新字段
		/*
		$options['_updateFieldValues'] = array(
				'modified_time' => date('Y-m-d H:i:s'),
		);*/
		
		$c_sql = self::_cb_row_save($table, $key, $file, $line, $fieldValues, $rowValues, $options);
	
		if(!$c_sql){
			//$this->_stats['fail']++;
			$this->_stats['fail'][] = $line;
		}else{
			$this->_stats['succ']++;
		}
	
		if(!empty($options['instantWrite'])){
			return $c_sql;
		}else{
			$sqls = array();
	
			if($c_sql){
				$sqls[] = $c_sql;
			}
			return implode(';', $sqls);
		}
	}
	
	/**
	 * 添加监听器
	 * @param mixed $listener 监听器
	 * @param string $type 监听类型
	 */
	public static function addListener($listener, $type){
		if(empty(self::$_listners)){
			self::$_listners = array();
		}
		
		if(empty(self::$_listners[$type])){
			self::$_listners[$type] = array();
		}
		
		self::$_listners[$type][] = $listener;
	}
	
	protected function _callListener($type, $data){
		if(empty(self::$_listners[$type])){
			return false;
		}
		
		foreach(self::$_listners[$type] as $listener){
			$this->_run_callback($listener, $data);
		}
	}
	
	public function archive_data(){
		$this->_log("=======================================");
		$this->_log("START Archive DATA");
		
		$driver = $this->_driver;
		if(!$driver){
			$this->_log("Driver cann't be empty", LOG::ERR);
			$this->_log("Stop Archive.");
			return false;
		}
		
		$this->_log("Driver: " . $driver);
		
		$table = self::getTable($driver);
		if(!$table){
			$this->_log("Cann't find table", LOG::ERR);
			$this->_log("Stop Archive.");
			return false;
		}
		
		$table_archive = $table . '_archive';
		
		$this->_log("Table: " . $table);
		$this->_log("Archive Table: " . $table_archive);
		
		$archiveDays = self::getArchiveDays($driver);
		if(!$archiveDays || $archiveDays<=0){
			$this->_log("Archive day less than 1 day", LOG::ERR);
			$this->_log("Stop Archive.");
			return false;
		}
		
		$this->_log("Archive Days: " . $archiveDays);
	
		import('ORG.Util.Debug');
	
		$debug_start_mark = 'start_archive_' . $driver;
		$debug_end_mark = 'end_archive_' . $driver;
		Debug::mark($debug_start_mark);
	
		$archiveDate = date('Y-m-d', strtotime('-' . $archiveDays . 'days'));
		
		$this->_log('Archive data before ' . $archiveDate);
		

		$mysqli = $this->_mysqli ();
	
		switch($driver){
			case self::SHSZ_DAY:
			case self::SHSZ_DAY_FORWARD:
			case self::HK_DAY:
			case self::HK_DAY_FORWARD:
				$insert_sql = "INSERT INTO {$table_archive} SELECT * FROM {$table} WHERE date<'{$archiveDate}'";
				$delete_sql = "DELETE FROM {$table} WHERE date<'{$archiveDate}'";
				break;
				
			case self::SHSZ_5MIN:
			case self::SHSZ_5MIN_FORWARD:
			case self::HK_5MIN:
			case self::HK_5MIN_FORWARD:
				$archiveDateTime = $archiveDate . ' 00:00:00';
				$insert_sql = "INSERT INTO {$table_archive} SELECT * FROM {$table} WHERE datetime<'{$archiveDateTime}'";
				$delete_sql = "DELETE FROM {$table} WHERE datetime<'{$archiveDateTime}'";
				break;
				
			default:
				$this->_log("Not supported driver");
				$this->_log("Stop Archive.");
				return false;
		}
		
		$repair_sql = "REPAIR TABLE {$table}";
		$optimize_sql = "OPTIMIZE TABLE {$table}";
		
		$this->_log("Insert SQL: " . $insert_sql, LOG::DEBUG);
		$this->_log("Delete SQL: " . $delete_sql, LOG::DEBUG);
		
		//return $insert_sql . "\n\n" . $delete_sql;
		
		
		$this->_log("Try to insert into archive table");
		if (false ==$mysqli->query ( $insert_sql )) {
			$this->_log ( sprintf ( 'Query Error(%s) %s', $mysqli->errno, $mysqli->error ), LOG::ERR );
			return false;
		}
		$inserted_rows = $mysqli->affected_rows;
		$this->_log("Insert Succ!");
		$this->_log("Inserted Rows: " . $inserted_rows);
		
		
		
		$this->_log("Try to delete from table");
		if (false ==$mysqli->query ( $delete_sql )) {
			$this->_log ( sprintf ( 'Query Error(%s) %s', $mysqli->errno, $mysqli->error ), LOG::ERR );
			return false;
		}
		$deleted_rows = $mysqli->affected_rows;
		$this->_log("Delete Succ!");
		$this->_log("Deleted Rows: " . $deleted_rows);
		
		
		$this->_log("Try to repair table");
		if (false ==$mysqli->query ( $repair_sql )) {
			$this->_log ( sprintf ( 'Query Error(%s) %s', $mysqli->errno, $mysqli->error ), LOG::ERR );
			return false;
		}
		$this->_log("Repair Succ!");
		
		
		$this->_log("Try to optimize table");
		if (false ==$mysqli->query ( $optimize_sql )) {
			$this->_log ( sprintf ( 'Query Error(%s) %s', $mysqli->errno, $mysqli->error ), LOG::ERR );
			return false;
		}
		$this->_log("Optimize Succ!");
	
		Debug::mark ( $debug_end_mark );
		$performance = "Cost time " . Debug::useTime ( $debug_start_mark, $debug_end_mark ) . 's';
		$performance .= ", used memory " . Debug::useMemory ( $debug_start_mark, $debug_end_mark ) . 'kb';
		$performance .= ", peak memory " . Debug::getMemPeak ( $debug_start_mark, $debug_end_mark ) . 'kb';
		$this->_log ( $performance );
	
		$this->_log ( "END ARCHIVE DATA" );
	
		return "Inserted Rows: {$inserted_rows}, Deleted Rows: {$deleted_rows}";
	}

}


SHSZRealtime.class.php  沪深实时数据更新

<?php defined('IN_HEAVEN') or die('Hacking Attempt!');
/**
 * 股票数据导入: 6-沪深实时数据
 * 
 * PHP version 5
 * 
 * @category	4SWeb
 * @package     Admin
 * @subpackage  Custom
 * @version     SVN: $Id: SHSZRealtime.class.php 48 2014-11-16 13:25:47Z blin.z $
 */
class SHSZRealtime extends StockImport{
	protected $_table = 'shsz_realtime';
	protected $_table_5min = 'shsz_realtime_5min';
	protected $_market = 'SHSZ';
	
	public function importFile($file, $file_content=NULL){
		$fields = array(
				'Time' => array('title' => 'Time', 'format' => 'l', 'length' => 4),
				'Symbol' => array('title' => 'Symbol', 'format' => '', 'length' => 12),
				'Name' => array('title' => 'Name', 'format' => '', 'length' => 16),
				
				'Price3' => array('title' => 'Price3', 'format' => 'f', 'length' => 4),
				'Vol2' => array('title' => 'Vol2', 'format' => 'f', 'length' => 4),
				'OpenInt' => array('title' => 'OpenInt', 'format' => 'f', 'length' => 4),
				'Price2' => array('title' => 'Price2', 'format' => 'f', 'length' => 4),
				'LastClose' => array('title' => 'LastClose', 'format' => 'f', 'length' => 4),
				'Open' => array('title' => 'Open', 'format' => 'f', 'length' => 4),
				'High' => array('title' => 'High', 'format' => 'f', 'length' => 4),
				'Low' => array('title' => 'Low', 'format' => 'f', 'length' => 4),
				'NewPrice' => array('title' => 'NewPrice', 'format' => 'f', 'length' => 4),
				'Volume' => array('title' => 'Volume', 'format' => 'f', 'length' => 4),
				'Amount' => array('title' => 'Amount', 'format' => 'f', 'length' => 4),
				'BP1' => array('title' => 'BP1', 'format' => 'f', 'length' => 4),
				'BP2' => array('title' => 'BP2', 'format' => 'f', 'length' => 4),
				'BP3' => array('title' => 'BP3', 'format' => 'f', 'length' => 4),
				'BP4' => array('title' => 'BP4', 'format' => 'f', 'length' => 4),
				'BP5' => array('title' => 'BP5', 'format' => 'f', 'length' => 4),
				'BV1' => array('title' => 'BV1', 'format' => 'f', 'length' => 4),
				'BV2' => array('title' => 'BV2', 'format' => 'f', 'length' => 4),
				'BV3' => array('title' => 'BV3', 'format' => 'f', 'length' => 4),
				'BV4' => array('title' => 'BV4', 'format' => 'f', 'length' => 4),
				'BV5' => array('title' => 'BV5', 'format' => 'f', 'length' => 4),
				'SP1' => array('title' => 'SP1', 'format' => 'f', 'length' => 4),
				'SP2' => array('title' => 'SP2', 'format' => 'f', 'length' => 4),
				'SP3' => array('title' => 'SP3', 'format' => 'f', 'length' => 4),
				'SP4' => array('title' => 'SP4', 'format' => 'f', 'length' => 4),
				'SP5' => array('title' => 'SP5', 'format' => 'f', 'length' => 4),
				'SV1' => array('title' => 'SV1', 'format' => 'f', 'length' => 4),
				'SV2' => array('title' => 'SV2', 'format' => 'f', 'length' => 4),
				'SV3' => array('title' => 'SV3', 'format' => 'f', 'length' => 4),
				'SV4' => array('title' => 'SV4', 'format' => 'f', 'length' => 4),
				'SV5' => array('title' => 'SV5', 'format' => 'f', 'length' => 4),
		);
	
		$options = array(
				'dataRow' => 1, // 数据开始行数
				'instantWrite' => false, // 实时写
				//'fieldCallback' => array($this, '_cb_field'),
				'rowCallback' => array($this, '_cb_row'),
	
				//'fromEncoding' => 'GBK',
				//'toEncoding' => 'UTF-8',
				
				'length' => 156,
		);
	
		return $this->_importFile($file, $fields, $options, $file_content);
	}
	
	public function runDownImport($truncate=false, $save=false){
		import('ORG.Util.Debug');
		
		$debug_start_mark = 'start_import_realtime';
		$debug_end_mark = 'end_import_realtime';
		
		Debug::mark($debug_start_mark);
		
		$this->_log("=======================================");
		$this->_log("START DOWN & IMPORT RealTime");
		
		$url = C('WSTOCK_REALTIME_URL.'.$this->_market);
		
		if(!$url){
			$this->_log("RealTime URL is empty.", LOG::ERR);
			$this->_log("Quit, Do Nothing.");
			return false;
		}
		
		Debug::mark('start_download_realtime_file');
		
		$this->_download_stamp = time();
		$this->_download_time = date('Y-m-d H:i:s');
		
		$this->_log("Try to get data from URL");
		$this->_log("Download Record Time: " . $this->_download_time);
		
		$this->_log("Request URL: " . $url, LOG::DEBUG);
				
		$content = $this->get($url);
		if(!$content){
			$this->_log("Get empty data from URL", LOG::ERR);
			$this->_log("Quit, Do Nothing.");
			return false;
		}else{
			$this->_log("Succ Down realtime file!");
		}
		
		Debug::mark('end_download_realtime_file');
		
		$down_secs = Debug::useTime('start_download_realtime_file','end_download_realtime_file');
		
		$this->_log("Cost time " . $down_secs .'s');
		
		// 是否超时?
		$interval = C('STOCK_REALTIME_UPDATE_INTERVAL');
		if($interval && $down_secs && ($down_secs>=($interval/2))){
			$this->_log("Skip Update!");
			$this->_log("Download cost time more than half interval time $interval/2", LOG::ERR);
			return false;
		}
		
		if(!self::_is_zip($content)){
			$this->_log("Server returned info package");
			$this->_log("Info: " . $content);
			$this->_log("Quit, Do Nothing.");
			return false;
		}
		
		$realtime_dir = $this->_get_realtime_dir(date('Y-m-d'), $save);
		
		list ( $usec, $sec ) = explode ( " ", microtime () );
		$microtime = trim(( float ) $usec + ( float ) $sec);
		
		if(!preg_match('/[\?&]m=([A-Z,]+)&?/', $url, $matches)){
			$this->_log("Invalid realtime URL: " . $url, LOG::ERR);
			$this->_log("Quit, Do Nothing.");
			return false;
		}
		
		$market = $matches[1];
		$realtime_file = $realtime_dir . 'wsRealtime_' . $market . '_' . $microtime . '.wsz';
		
		if($save){
			if(!file_put_contents($realtime_file, $content)){
				$this->_log("Cann't Save realtime file: " . $realtime_file, LOG::ERR);
				$this->_log("Quit, Do Nothing.");
				return false;
			}else{
				$this->_log("Succ Saved realtime file as " . $realtime_file);
			}
		}

		if($truncate){
			$this->_log("Truncate table before import");
			$succ = $this->_VM()->execute('TRUNCATE TABLE `' . $this->_table . "`");
			$this->_log($succ!==false?"Succ":"Fail: " . $this->_VM()->getDbError ());
		}
		
		$this->importFile($realtime_file, $content);

		Debug::mark($debug_end_mark);
		
		$performance = "Cost time " . Debug::useTime($debug_start_mark,$debug_end_mark).'s';
		$performance .= ", used memory " . Debug::useMemory($debug_start_mark,$debug_end_mark).'kb';
		$performance .= ", peak memory " . Debug::getMemPeak($debug_start_mark, $debug_end_mark).'kb';
		$this->_log( $performance );
		
		$this->_log("END IMPORT RealTime.");
	}
	
	public function _cb_field($file, $line, $column, $field, $value, $options=NULL){
		$value = self::filter_trim($value);
		switch($field){
			// 股票代码
			case 'Symbol':
				$value = String::autoCharset($value, 'gbk', C('DEFAULT_CHARSET'));
				$value = trim($value);
				$v = self::filter_stock_code($value);
				if(!$value){
					return true;
				}
				break;
				
			// 名称
			case 'Name':
				$value = String::autoCharset($value, 'gbk', C('DEFAULT_CHARSET'));
				$value = trim($value);
				break;
	
			// 时间转换
			case 'Time':
				//$v = self::filter_date($value);
				//$v = date('Y-m-d H:i:s', $v);
				if(!$value){
					return true;
				}
				break;
				
			case 'Price3':
			case 'Vol2':
			case 'OpenInt':
			case 'Price2':
			case 'LastClose':
			case 'Open':
			case 'High':
			case 'Low':
			case 'NewPrice':
			case 'Volume':
			case 'Amount':
			case 'BP1':
			case 'BP2':
			case 'BP3':
			case 'BP4':
			case 'BP5':
			case 'BV1':
			case 'BV2':
			case 'BV3':
			case 'BV4':
			case 'BV5':
			case 'SP1':
			case 'SP2':
			case 'SP3':
			case 'SP4':
			case 'SP5':
			case 'SV1':
			case 'SV2':
			case 'SV3':
			case 'SV4':
			case 'SV5':
				$value = sprintf('%.3f', $value);
				break;	
		}
	
		return $value;
	}
	
	public function _cb_row($file, $line, $fieldValues, $rowValues, $options=NULL){
		$this->_stats['total']++;
	
		$fieldValues['Symbol'] = trim(mb_convert_encoding( $fieldValues['Symbol'], 'utf-8', 'gbk' ));
		$fieldValues['Name'] = trim(mb_convert_encoding( $fieldValues['Name'], 'utf-8', 'gbk' ));
		
		static $float_fields = array(
				'Price3' ,
				'Vol2' ,
				'OpenInt' ,
				'Price2' ,
				'LastClose' ,
				'Open' ,
				'High' ,
				'Low' ,
				'NewPrice' ,
				'Volume' ,
				'Amount' ,
				'BP1' ,
				'BP2' ,
				'BP3' ,
				'BP4' ,
				'BP5' ,
				'BV1' ,
				'BV2' ,
				'BV3' ,
				'BV4' ,
				'BV5' ,
				'SP1' ,
				'SP2' ,
				'SP3' ,
				'SP4' ,
				'SP5' ,
				'SV1' ,
				'SV2' ,
				'SV3' ,
				'SV4' ,
				'SV5' ,
		);
		
		foreach($float_fields as $f){
			if(isset($fieldValues[$f])){
				$fieldValues[$f] = sprintf ( '%.3f', $fieldValues[$f] );
			}
		}

		$fieldValues['recordtime'] = $this->_download_time;
		$sql = SQLKit::replace_stmt($this->_table, $fieldValues);
		
		$this->_stats['succ']++;
		
		return $sql;
	}
	
	/**
	 * 判断是否为压缩文件
	 * @param string $content
	 * @return boolean
	 */
	protected function _is_zip($content){
		$tagstr = substr($content, 0, 4);
		$tagnum = current(unpack("l", $tagstr));
		return ($tagnum>16777216 || $tagnum<=0)?false:true;
	}
	
	protected function _decode($content){
		$length = strlen($content);
		if(!$length){
			return $this->_trigger_error(
					"File content is empty"
					, self::ERR_EMPTY_DATA
					, null
					, __FILE__, __LINE__
			);
		}
		
		$readpos = 0;
		$rawdata = '';
		while($readpos<$length){
			//$blocksize = $this->_readcontent($content, $this->_readpos, 4);
			$blocksize = substr($content, $readpos, 4);
			$readpos += 4;
			
			$blocksize= current(unpack("l", $blocksize));
			
			if(!$blocksize || $blocksize<=0){
				return $this->_trigger_error(
						"Block size is zero or negative"
						, self::ERR_FILE_READ
						, array('readpos'=>$readpos)
						, __FILE__, __LINE__
				);
			}
			
			$zipsize = $blocksize - 4;
			if ($zipsize <= 0) {
				return $this->_trigger_error ( 
						"Zip size is zero or negative"
						, self::ERR_FILE_READ
						, array (
							'readpos' => $readpos
						)
						, __FILE__, __LINE__ );
			}
			
			//$orisize = $this->_readcontent($content, $this->_readpos, 4);
			$orisize = substr($content, $readpos, 4);
			$readpos += 4;
			
			$orisize= current(unpack("l", $orisize));
			if ($orisize <= 0) {
				return $this->_trigger_error (
						"Ori size is zero or negative"
						, self::ERR_FILE_READ
						, array (
								'readpos' => $this->_readpos
						)
						, __FILE__, __LINE__ );
			}
			
			//$zipbody = $this->_readcontent($content, $this->_readpos, $zipsize);
			$zipbody = substr($content, $readpos, $zipsize);
			$readpos += $zipsize;
			
			if (!$zipbody) {
				return $this->_trigger_error (
						"Zip body is empty"
						, self::ERR_FILE_READ
						, array (
								'readpos' => $this->_readpos
						)
						, __FILE__, __LINE__ );
			}
			
			$zipbody = zlib_decode($zipbody);
			if (!$zipbody) {
				return $this->_trigger_error (
						"Cann't decode zip body"
						, self::ERR_FILE_READ
						, array (
								'readpos' => $this->_readpos
						)
						, __FILE__, __LINE__ );
			}
			
			$rawdata .= $zipbody;
		}
		
		return $rawdata;
	}
	
	protected function _readcontent($content, $start, $length){
		$this->_readpos += $length;
		return substr($content, $start, $length);
	}

	protected function _import($file, $fields, $options=array(), $content=NULL){
		$default_options = array(
				'titleRow' => false, //  标题行数,设置才会检测Excel标题列是否符合要求
				'dataRow' => 1, // 数据开始行数,可以通过传数组方式设置读取的开始和终止行数,格式如array(开始行数,终止行数)
				'rowCallback' => NULL, // 行数据处理回调,传参数:$row, $fieldValues, $rowValues, $options
				'fieldCallback' => NULL, // 字段过滤处理回调,传参数:$column, $field, $value, $options
	
				'length' => NULL, // 行长度
	
				'fromEncoding' => NULL, // 源文件编码
				'toEncoding' => NULL, // 数据编码
	
		);
	
		$options = array_merge($default_options, $options);
	
		$aColumnMaps = array ( 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'AA', 'AB', 'AC', 'AD', 'AE', 'AF', 'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', 'AO', 'AP', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AV', 'AW', 'AX', 'AY', 'AZ', 'BA', 'BB', 'BC', 'BD', 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ', 'BK', 'BL', 'BM', 'BN', 'BO', 'BP', 'BQ', 'BR', 'BS', 'BT', 'BU', 'BV', 'BW', 'BX', 'BY', 'BZ', 'CA', 'CB', 'CC', 'CD', 'CE', 'CF', 'CG', 'CH', 'CI', 'CJ', 'CK', 'CL', 'CM', 'CN', 'CO', 'CP', 'CQ', 'CR', 'CS', 'CT', 'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', 'DA', 'DB', 'DC', 'DD', 'DE', 'DF', 'DG', 'DH', 'DI', 'DJ', 'DK', 'DL', 'DM', 'DN', 'DO', 'DP', 'DQ', 'DR', 'DS', 'DT', 'DU', 'DV', 'DW', 'DX', 'DY', 'DZ', 'EA', 'EB', 'EC', 'ED', 'EE', 'EF', 'EG', 'EH', 'EI', 'EJ', 'EK', 'EL', 'EM', 'EN', 'EO', 'EP', 'EQ', 'ER', 'ES', 'ET', 'EU', 'EV', 'EW', 'EX', 'EY', 'EZ', 'FA', 'FB', 'FC', 'FD', 'FE', 'FF', 'FG', 'FH', 'FI', 'FJ', 'FK', 'FL', 'FM', 'FN', 'FO', 'FP', 'FQ', 'FR', 'FS', 'FT', 'FU', 'FV', 'FW', 'FX', 'FY', 'FZ', 'GA', 'GB', 'GC', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GJ', 'GK', 'GL', 'GM', 'GN', 'GO', 'GP', 'GQ', 'GR', 'GS', 'GT', 'GU', 'GV', 'GW', 'GX', 'GY', 'GZ', 'HA', 'HB', 'HC', 'HD', 'HE', 'HF', 'HG', 'HH', 'HI', 'HJ', 'HK', 'HL', 'HM', 'HN', 'HO', 'HP', 'HQ', 'HR', 'HS', 'HT', 'HU', 'HV', 'HW', 'HX', 'HY', 'HZ', 'IA', 'IB', 'IC', 'ID', 'IE', 'IF', 'IG', 'IH', 'II', 'IJ', 'IK', 'IL', 'IM', 'IN', 'IO', 'IP', 'IQ', 'IR', 'IS', 'IT', 'IU', 'IV', 'IW', 'IX', 'IY', 'IZ', 'JA', 'JB', 'JC', 'JD', 'JE', 'JF', 'JG', 'JH', 'JI', 'JJ', 'JK', 'JL', 'JM', 'JN', 'JO', 'JP', 'JQ', 'JR', 'JS', 'JT', 'JU', 'JV', 'JW', 'JX', 'JY', 'JZ', 'KA', 'KB', 'KC', 'KD', 'KE', 'KF', 'KG', 'KH', 'KI', 'KJ', 'KK', 'KL', 'KM', 'KN', 'KO', 'KP', 'KQ', 'KR', 'KS', 'KT', 'KU', 'KV', 'KW', 'KX', 'KY', 'KZ', 'LA', 'LB', 'LC', 'LD', 'LE', 'LF', 'LG', 'LH', 'LI', 'LJ', 'LK', 'LL', 'LM', 'LN', 'LO', 'LP', 'LQ', 'LR', 'LS', 'LT', 'LU', 'LV', 'LW', 'LX', 'LY', 'LZ', 'MA', 'MB', 'MC', 'MD', 'ME', 'MF', 'MG', 'MH', 'MI', 'MJ', 'MK', 'ML', 'MM', 'MN', 'MO', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', 'MV', 'MW', 'MX', 'MY', 'MZ', 'NA', 'NB', 'NC', 'ND', 'NE', 'NF', 'NG', 'NH', 'NI', 'NJ', 'NK', 'NL', 'NM', 'NN', 'NO', 'NP', 'NQ', 'NR', 'NS', 'NT', 'NU', 'NV', 'NW', 'NX', 'NY', 'NZ', 'OA', 'OB', 'OC', 'OD', 'OE', 'OF', 'OG', 'OH', 'OI', 'OJ', 'OK', 'OL', 'OM', 'ON', 'OO', 'OP', 'OQ', 'OR', 'OS', 'OT', 'OU', 'OV', 'OW', 'OX', 'OY', 'OZ', 'PA', 'PB', 'PC', 'PD', 'PE', 'PF', 'PG', 'PH', 'PI', 'PJ', 'PK', 'PL', 'PM', 'PN', 'PO', 'PP', 'PQ', 'PR', 'PS', 'PT', 'PU', 'PV', 'PW', 'PX', 'PY', 'PZ', 'QA', 'QB', 'QC', 'QD', 'QE', 'QF', 'QG', 'QH', 'QI', 'QJ', 'QK', 'QL', 'QM', 'QN', 'QO', 'QP', 'QQ', 'QR', 'QS', 'QT', 'QU', 'QV', 'QW', 'QX', 'QY', 'QZ', 'RA', 'RB', 'RC', 'RD', 'RE', 'RF', 'RG', 'RH', 'RI', 'RJ', 'RK', 'RL', 'RM', 'RN', 'RO', 'RP', 'RQ', 'RR', 'RS', 'RT', 'RU', 'RV', 'RW', 'RX', 'RY', 'RZ', 'SA', 'SB', 'SC', 'SD', 'SE', 'SF', 'SG', 'SH', 'SI', 'SJ', 'SK', 'SL', 'SM', 'SN', 'SO', 'SP', 'SQ', 'SR', 'SS', 'ST', 'SU', 'SV', 'SW', 'SX', 'SY', 'SZ', 'TA', 'TB', 'TC', 'TD', 'TE', 'TF', 'TG', 'TH', 'TI', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO', 'TP', 'TQ', 'TR', 'TS', 'TT', 'TU', 'TV', 'TW', 'TX', 'TY', 'TZ', 'UA', 'UB', 'UC', 'UD', 'UE', 'UF', 'UG', 'UH', 'UI', 'UJ', 'UK', 'UL', 'UM', 'UN', 'UO', 'UP', 'UQ', 'UR', 'US', 'UT', 'UU', 'UV', 'UW', 'UX', 'UY', 'UZ', 'VA', 'VB', 'VC', 'VD', 'VE', 'VF', 'VG', 'VH', 'VI', 'VJ', 'VK', 'VL', 'VM', 'VN', 'VO', 'VP', 'VQ', 'VR', 'VS', 'VT', 'VU', 'VV', 'VW', 'VX', 'VY', 'VZ', 'WA', 'WB', 'WC', 'WD', 'WE', 'WF', 'WG', 'WH', 'WI', 'WJ', 'WK', 'WL', 'WM', 'WN', 'WO', 'WP', 'WQ', 'WR', 'WS', 'WT', 'WU', 'WV', 'WW', 'WX', 'WY', 'WZ', 'XA', 'XB', 'XC', 'XD', 'XE', 'XF', 'XG', 'XH', 'XI', 'XJ', 'XK', 'XL', 'XM', 'XN', 'XO', 'XP', 'XQ', 'XR', 'XS', 'XT', 'XU', 'XV', 'XW', 'XX', 'XY', 'XZ', 'YA', 'YB', 'YC', 'YD', 'YE', 'YF', 'YG', 'YH', 'YI', 'YJ', 'YK', 'YL', 'YM', 'YN', 'YO', 'YP', 'YQ', 'YR', 'YS', 'YT', 'YU', 'YV', 'YW', 'YX', 'YY', 'YZ', 'ZA', 'ZB', 'ZC', 'ZD', 'ZE', 'ZF', 'ZG', 'ZH', 'ZI', 'ZJ', 'ZK', 'ZL', 'ZM', 'ZN', 'ZO', 'ZP', 'ZQ', 'ZR', 'ZS', 'ZT', 'ZU', 'ZV', 'ZW', 'ZX', 'ZY', 'ZZ' );
		$aColumnFlipMaps = array ( 'A' => '0', 'B' => 1, 'C' => 2, 'D' => 3, 'E' => 4, 'F' => 5, 'G' => 6, 'H' => 7, 'I' => 8, 'J' => 9, 'K' => 10, 'L' => 11, 'M' => 12, 'N' => 13, 'O' => 14, 'P' => 15, 'Q' => 16, 'R' => 17, 'S' => 18, 'T' => 19, 'U' => 20, 'V' => 21, 'W' => 22, 'X' => 23, 'Y' => 24, 'Z' => 25, 'AA' => 26, 'AB' => 27, 'AC' => 28, 'AD' => 29, 'AE' => 30, 'AF' => 31, 'AG' => 32, 'AH' => 33, 'AI' => 34, 'AJ' => 35, 'AK' => 36, 'AL' => 37, 'AM' => 38, 'AN' => 39, 'AO' => 40, 'AP' => 41, 'AQ' => 42, 'AR' => 43, 'AS' => 44, 'AT' => 45, 'AU' => 46, 'AV' => 47, 'AW' => 48, 'AX' => 49, 'AY' => 50, 'AZ' => 51, 'BA' => 52, 'BB' => 53, 'BC' => 54, 'BD' => 55, 'BE' => 56, 'BF' => 57, 'BG' => 58, 'BH' => 59, 'BI' => 60, 'BJ' => 61, 'BK' => 62, 'BL' => 63, 'BM' => 64, 'BN' => 65, 'BO' => 66, 'BP' => 67, 'BQ' => 68, 'BR' => 69, 'BS' => 70, 'BT' => 71, 'BU' => 72, 'BV' => 73, 'BW' => 74, 'BX' => 75, 'BY' => 76, 'BZ' => 77, 'CA' => 78, 'CB' => 79, 'CC' => 80, 'CD' => 81, 'CE' => 82, 'CF' => 83, 'CG' => 84, 'CH' => 85, 'CI' => 86, 'CJ' => 87, 'CK' => 88, 'CL' => 89, 'CM' => 90, 'CN' => 91, 'CO' => 92, 'CP' => 93, 'CQ' => 94, 'CR' => 95, 'CS' => 96, 'CT' => 97, 'CU' => 98, 'CV' => 99, 'CW' => 100, 'CX' => 101, 'CY' => 102, 'CZ' => 103, 'DA' => 104, 'DB' => 105, 'DC' => 106, 'DD' => 107, 'DE' => 108, 'DF' => 109, 'DG' => 110, 'DH' => 111, 'DI' => 112, 'DJ' => 113, 'DK' => 114, 'DL' => 115, 'DM' => 116, 'DN' => 117, 'DO' => 118, 'DP' => 119, 'DQ' => 120, 'DR' => 121, 'DS' => 122, 'DT' => 123, 'DU' => 124, 'DV' => 125, 'DW' => 126, 'DX' => 127, 'DY' => 128, 'DZ' => 129, 'EA' => 130, 'EB' => 131, 'EC' => 132, 'ED' => 133, 'EE' => 134, 'EF' => 135, 'EG' => 136, 'EH' => 137, 'EI' => 138, 'EJ' => 139, 'EK' => 140, 'EL' => 141, 'EM' => 142, 'EN' => 143, 'EO' => 144, 'EP' => 145, 'EQ' => 146, 'ER' => 147, 'ES' => 148, 'ET' => 149, 'EU' => 150, 'EV' => 151, 'EW' => 152, 'EX' => 153, 'EY' => 154, 'EZ' => 155, 'FA' => 156, 'FB' => 157, 'FC' => 158, 'FD' => 159, 'FE' => 160, 'FF' => 161, 'FG' => 162, 'FH' => 163, 'FI' => 164, 'FJ' => 165, 'FK' => 166, 'FL' => 167, 'FM' => 168, 'FN' => 169, 'FO' => 170, 'FP' => 171, 'FQ' => 172, 'FR' => 173, 'FS' => 174, 'FT' => 175, 'FU' => 176, 'FV' => 177, 'FW' => 178, 'FX' => 179, 'FY' => 180, 'FZ' => 181, 'GA' => 182, 'GB' => 183, 'GC' => 184, 'GD' => 185, 'GE' => 186, 'GF' => 187, 'GG' => 188, 'GH' => 189, 'GI' => 190, 'GJ' => 191, 'GK' => 192, 'GL' => 193, 'GM' => 194, 'GN' => 195, 'GO' => 196, 'GP' => 197, 'GQ' => 198, 'GR' => 199, 'GS' => 200, 'GT' => 201, 'GU' => 202, 'GV' => 203, 'GW' => 204, 'GX' => 205, 'GY' => 206, 'GZ' => 207, 'HA' => 208, 'HB' => 209, 'HC' => 210, 'HD' => 211, 'HE' => 212, 'HF' => 213, 'HG' => 214, 'HH' => 215, 'HI' => 216, 'HJ' => 217, 'HK' => 218, 'HL' => 219, 'HM' => 220, 'HN' => 221, 'HO' => 222, 'HP' => 223, 'HQ' => 224, 'HR' => 225, 'HS' => 226, 'HT' => 227, 'HU' => 228, 'HV' => 229, 'HW' => 230, 'HX' => 231, 'HY' => 232, 'HZ' => 233, 'IA' => 234, 'IB' => 235, 'IC' => 236, 'ID' => 237, 'IE' => 238, 'IF' => 239, 'IG' => 240, 'IH' => 241, 'II' => 242, 'IJ' => 243, 'IK' => 244, 'IL' => 245, 'IM' => 246, 'IN' => 247, 'IO' => 248, 'IP' => 249, 'IQ' => 250, 'IR' => 251, 'IS' => 252, 'IT' => 253, 'IU' => 254, 'IV' => 255, 'IW' => 256, 'IX' => 257, 'IY' => 258, 'IZ' => 259, 'JA' => 260, 'JB' => 261, 'JC' => 262, 'JD' => 263, 'JE' => 264, 'JF' => 265, 'JG' => 266, 'JH' => 267, 'JI' => 268, 'JJ' => 269, 'JK' => 270, 'JL' => 271, 'JM' => 272, 'JN' => 273, 'JO' => 274, 'JP' => 275, 'JQ' => 276, 'JR' => 277, 'JS' => 278, 'JT' => 279, 'JU' => 280, 'JV' => 281, 'JW' => 282, 'JX' => 283, 'JY' => 284, 'JZ' => 285, 'KA' => 286, 'KB' => 287, 'KC' => 288, 'KD' => 289, 'KE' => 290, 'KF' => 291, 'KG' => 292, 'KH' => 293, 'KI' => 294, 'KJ' => 295, 'KK' => 296, 'KL' => 297, 'KM' => 298, 'KN' => 299, 'KO' => 300, 'KP' => 301, 'KQ' => 302, 'KR' => 303, 'KS' => 304, 'KT' => 305, 'KU' => 306, 'KV' => 307, 'KW' => 308, 'KX' => 309, 'KY' => 310, 'KZ' => 311, 'LA' => 312, 'LB' => 313, 'LC' => 314, 'LD' => 315, 'LE' => 316, 'LF' => 317, 'LG' => 318, 'LH' => 319, 'LI' => 320, 'LJ' => 321, 'LK' => 322, 'LL' => 323, 'LM' => 324, 'LN' => 325, 'LO' => 326, 'LP' => 327, 'LQ' => 328, 'LR' => 329, 'LS' => 330, 'LT' => 331, 'LU' => 332, 'LV' => 333, 'LW' => 334, 'LX' => 335, 'LY' => 336, 'LZ' => 337, 'MA' => 338, 'MB' => 339, 'MC' => 340, 'MD' => 341, 'ME' => 342, 'MF' => 343, 'MG' => 344, 'MH' => 345, 'MI' => 346, 'MJ' => 347, 'MK' => 348, 'ML' => 349, 'MM' => 350, 'MN' => 351, 'MO' => 352, 'MP' => 353, 'MQ' => 354, 'MR' => 355, 'MS' => 356, 'MT' => 357, 'MU' => 358, 'MV' => 359, 'MW' => 360, 'MX' => 361, 'MY' => 362, 'MZ' => 363, 'NA' => 364, 'NB' => 365, 'NC' => 366, 'ND' => 367, 'NE' => 368, 'NF' => 369, 'NG' => 370, 'NH' => 371, 'NI' => 372, 'NJ' => 373, 'NK' => 374, 'NL' => 375, 'NM' => 376, 'NN' => 377, 'NO' => 378, 'NP' => 379, 'NQ' => 380, 'NR' => 381, 'NS' => 382, 'NT' => 383, 'NU' => 384, 'NV' => 385, 'NW' => 386, 'NX' => 387, 'NY' => 388, 'NZ' => 389, 'OA' => 390, 'OB' => 391, 'OC' => 392, 'OD' => 393, 'OE' => 394, 'OF' => 395, 'OG' => 396, 'OH' => 397, 'OI' => 398, 'OJ' => 399, 'OK' => 400, 'OL' => 401, 'OM' => 402, 'ON' => 403, 'OO' => 404, 'OP' => 405, 'OQ' => 406, 'OR' => 407, 'OS' => 408, 'OT' => 409, 'OU' => 410, 'OV' => 411, 'OW' => 412, 'OX' => 413, 'OY' => 414, 'OZ' => 415, 'PA' => 416, 'PB' => 417, 'PC' => 418, 'PD' => 419, 'PE' => 420, 'PF' => 421, 'PG' => 422, 'PH' => 423, 'PI' => 424, 'PJ' => 425, 'PK' => 426, 'PL' => 427, 'PM' => 428, 'PN' => 429, 'PO' => 430, 'PP' => 431, 'PQ' => 432, 'PR' => 433, 'PS' => 434, 'PT' => 435, 'PU' => 436, 'PV' => 437, 'PW' => 438, 'PX' => 439, 'PY' => 440, 'PZ' => 441, 'QA' => 442, 'QB' => 443, 'QC' => 444, 'QD' => 445, 'QE' => 446, 'QF' => 447, 'QG' => 448, 'QH' => 449, 'QI' => 450, 'QJ' => 451, 'QK' => 452, 'QL' => 453, 'QM' => 454, 'QN' => 455, 'QO' => 456, 'QP' => 457, 'QQ' => 458, 'QR' => 459, 'QS' => 460, 'QT' => 461, 'QU' => 462, 'QV' => 463, 'QW' => 464, 'QX' => 465, 'QY' => 466, 'QZ' => 467, 'RA' => 468, 'RB' => 469, 'RC' => 470, 'RD' => 471, 'RE' => 472, 'RF' => 473, 'RG' => 474, 'RH' => 475, 'RI' => 476, 'RJ' => 477, 'RK' => 478, 'RL' => 479, 'RM' => 480, 'RN' => 481, 'RO' => 482, 'RP' => 483, 'RQ' => 484, 'RR' => 485, 'RS' => 486, 'RT' => 487, 'RU' => 488, 'RV' => 489, 'RW' => 490, 'RX' => 491, 'RY' => 492, 'RZ' => 493, 'SA' => 494, 'SB' => 495, 'SC' => 496, 'SD' => 497, 'SE' => 498, 'SF' => 499, 'SG' => 500, 'SH' => 501, 'SI' => 502, 'SJ' => 503, 'SK' => 504, 'SL' => 505, 'SM' => 506, 'SN' => 507, 'SO' => 508, 'SP' => 509, 'SQ' => 510, 'SR' => 511, 'SS' => 512, 'ST' => 513, 'SU' => 514, 'SV' => 515, 'SW' => 516, 'SX' => 517, 'SY' => 518, 'SZ' => 519, 'TA' => 520, 'TB' => 521, 'TC' => 522, 'TD' => 523, 'TE' => 524, 'TF' => 525, 'TG' => 526, 'TH' => 527, 'TI' => 528, 'TJ' => 529, 'TK' => 530, 'TL' => 531, 'TM' => 532, 'TN' => 533, 'TO' => 534, 'TP' => 535, 'TQ' => 536, 'TR' => 537, 'TS' => 538, 'TT' => 539, 'TU' => 540, 'TV' => 541, 'TW' => 542, 'TX' => 543, 'TY' => 544, 'TZ' => 545, 'UA' => 546, 'UB' => 547, 'UC' => 548, 'UD' => 549, 'UE' => 550, 'UF' => 551, 'UG' => 552, 'UH' => 553, 'UI' => 554, 'UJ' => 555, 'UK' => 556, 'UL' => 557, 'UM' => 558, 'UN' => 559, 'UO' => 560, 'UP' => 561, 'UQ' => 562, 'UR' => 563, 'US' => 564, 'UT' => 565, 'UU' => 566, 'UV' => 567, 'UW' => 568, 'UX' => 569, 'UY' => 570, 'UZ' => 571, 'VA' => 572, 'VB' => 573, 'VC' => 574, 'VD' => 575, 'VE' => 576, 'VF' => 577, 'VG' => 578, 'VH' => 579, 'VI' => 580, 'VJ' => 581, 'VK' => 582, 'VL' => 583, 'VM' => 584, 'VN' => 585, 'VO' => 586, 'VP' => 587, 'VQ' => 588, 'VR' => 589, 'VS' => 590, 'VT' => 591, 'VU' => 592, 'VV' => 593, 'VW' => 594, 'VX' => 595, 'VY' => 596, 'VZ' => 597, 'WA' => 598, 'WB' => 599, 'WC' => 600, 'WD' => 601, 'WE' => 602, 'WF' => 603, 'WG' => 604, 'WH' => 605, 'WI' => 606, 'WJ' => 607, 'WK' => 608, 'WL' => 609, 'WM' => 610, 'WN' => 611, 'WO' => 612, 'WP' => 613, 'WQ' => 614, 'WR' => 615, 'WS' => 616, 'WT' => 617, 'WU' => 618, 'WV' => 619, 'WW' => 620, 'WX' => 621, 'WY' => 622, 'WZ' => 623, 'XA' => 624, 'XB' => 625, 'XC' => 626, 'XD' => 627, 'XE' => 628, 'XF' => 629, 'XG' => 630, 'XH' => 631, 'XI' => 632, 'XJ' => 633, 'XK' => 634, 'XL' => 635, 'XM' => 636, 'XN' => 637, 'XO' => 638, 'XP' => 639, 'XQ' => 640, 'XR' => 641, 'XS' => 642, 'XT' => 643, 'XU' => 644, 'XV' => 645, 'XW' => 646, 'XX' => 647, 'XY' => 648, 'XZ' => 649, 'YA' => 650, 'YB' => 651, 'YC' => 652, 'YD' => 653, 'YE' => 654, 'YF' => 655, 'YG' => 656, 'YH' => 657, 'YI' => 658, 'YJ' => 659, 'YK' => 660, 'YL' => 661, 'YM' => 662, 'YN' => 663, 'YO' => 664, 'YP' => 665, 'YQ' => 666, 'YR' => 667, 'YS' => 668, 'YT' => 669, 'YU' => 670, 'YV' => 671, 'YW' => 672, 'YX' => 673, 'YY' => 674, 'YZ' => 675, 'ZA' => 676, 'ZB' => 677, 'ZC' => 678, 'ZD' => 679, 'ZE' => 680, 'ZF' => 681, 'ZG' => 682, 'ZH' => 683, 'ZI' => 684, 'ZJ' => 685, 'ZK' => 686, 'ZL' => 687, 'ZM' => 688, 'ZN' => 689, 'ZO' => 690, 'ZP' => 691, 'ZQ' => 692, 'ZR' => 693, 'ZS' => 694, 'ZT' => 695, 'ZU' => 696, 'ZV' => 697, 'ZW' => 698, 'ZX' => 699, 'ZY' => 700, 'ZZ' => 701 );
	
		$aReturn = array();
		$aReturn5Min = array();
	
		if(!$options['length']){
			$options['length'] = 0;
			foreach($fields as $f){
				$options['length'] += $f['length'];
			}
		}
		
		//$this->_log("Block length: " . $options['length']);
		
		if(!$content){
			$content = file_get_contents($file);
		}
		
		if($content===false){
			return $this->_trigger_error(
					"Cann't open file: " . $file
					, self::ERR_FILE_READ
					, $file
					, __FILE__, __LINE__
			);
		}
		
		Debug::mark('start_decode_realtime_file');
		
		$this->_log("Try to decode data");
		
		$rawdata = $this->_decode($content);
		//$rawdata = $content;
		
		if(!$rawdata){
			return $this->_trigger_error("Cann't decode data"
					, self::ERR_FILE_READ
					, $file
					, __FILE__, __LINE__
			);
		}
		
		$this->_log("Succ decoded!");
		
		Debug::mark('end_decode_realtime_file');
		
		$this->_log("Cost time " . Debug::useTime('start_decode_realtime_file','end_decode_realtime_file').'s');
		
		$length = strlen($rawdata);
		$this->_readpos = 0;
		
		static $float_fields = array(
				'Price3' ,
				'Vol2' ,
				'OpenInt' ,
				'Price2' ,
				'LastClose' ,
				'Open' ,
				'High' ,
				'Low' ,
				'NewPrice' ,
				'Volume' ,
				'Amount' ,
				'BP1' ,
				'BP2' ,
				'BP3' ,
				'BP4' ,
				'BP5' ,
				'BV1' ,
				'BV2' ,
				'BV3' ,
				'BV4' ,
				'BV5' ,
				'SP1' ,
				'SP2' ,
				'SP3' ,
				'SP4' ,
				'SP5' ,
				'SV1' ,
				'SV2' ,
				'SV3' ,
				'SV4' ,
				'SV5' ,
		);
		
		Debug::mark('start_process_realtime_data');
		
		$this->_log("Try to process data");
	
		$line = 0; // 当前行数
		$readpos = 0; // 当前文件读取位置
		for ($i=0;$i<$length;$i+=$options['length']) {
			$line++; // 当前行数+1
			
			$fieldValues = array(); // 行字段值
	
			// 行原始数据
			$rowdata = substr($rawdata, $readpos, $options['length']);
			$readpos += $options['length'];
			
			// 行数据处理
			$rowpos = 0; // 当前行读取位置

			$fieldValues['Time'] = current(unpack('l', substr($rowdata, $rowpos, 4)));
			$rowpos += 4;
			
			$fieldValues['Symbol'] = trim(substr($rowdata, $rowpos, 12));
			$rowpos += 12;

			$fieldValues['Name'] = substr($rowdata, $rowpos, 16);
			$rowpos += 16;
	
			//$fieldValues['Symbol'] = trim(mb_convert_encoding( $fieldValues['Symbol'], 'utf-8', 'gbk' ));
			$fieldValues['Name'] = trim(mb_convert_encoding( $fieldValues['Name'], 'utf-8', 'gbk' ));
			
			foreach($float_fields as $f){
				$fieldValues[$f] = current(unpack('f', substr($rowdata, $rowpos, 4)));
				$rowpos += 4;
				
				$fieldValues[$f] = sprintf ( '%.3f', $fieldValues[$f] );
			}
				
			$fieldValues['recordtime'] = $this->_download_time;
			$aReturn[$line] = SQLKit::replace_stmt($this->_table, $fieldValues);
			$aReturn5Min[$line] = SQLKit::replace_stmt($this->_table_5min, $fieldValues);
				
			$this->_stats['total']++;
			$this->_stats['succ']++;
		}
		
		$this->_log("End process realtime data");
		
		Debug::mark('end_process_realtime_data');

		$this->_log("Cost time " . Debug::useTime('start_process_realtime_data','end_process_realtime_data').'s');

		Debug::mark('start_save_realtime_data');
		$this->_log("Try to save realtime data");
		
		
		// 写入内存
		/*
		$oCache = Cache::getInstance();
		$cache_id = 'SHSZ_REALTIME_DATA';
		$cache_expire = 3600;
		$cache_data = array(
				'serialized' => 0,
				'created' => time(),
				'expire' => $cache_expire,
				'data' => $aReturn,
		);	
		$oCache->set($cache_id, $cache_data, $cache_expire);*/
		
		// 写入数据库
		$mysqli = $this->_mysqli();
		
		// 检测是否已有更新
		$interval = C('STOCK_REALTIME_UPDATE_INTERVAL');
		if($interval && $interval>0 && (time()-$this->_download_stamp)>=$interval && $fieldValues['Time']){
			/* Select queries return a resultset */
			$result = $mysqli->query ( "Select Time FROM " . $this->_table . " LIMIT 1" );
			if ($result) {
				$row = $result->fetch_assoc ();
				
				/* free result set */
				$result->close ();
				
				if ($row && $row ['Time'] && $row ['Time'] > $fieldValues ['Time']) {
					$this->_log("Skip Update!");
					$this->_setError("Exist record is newer (" . date ( 'Y-m-d H:i:s', $row ['Time'] ) . ") than current download data (" . date ( 'Y-m-d H:i:s', $fieldValues ['Time'] ) . ")");
					return false;
				}
			}
		}

		//$chunk_sqls = array_chunk($aReturn, 1000);
		$chunk_sqls = array($aReturn);
		foreach ( $chunk_sqls as $sqls ) {
			$query = implode(';', $sqls);
			
			/* execute multi query */
			if (false == $mysqli->multi_query ( $query )) {
				$error = 'Query Error';
				$error .= ' (' . $mysqli->errno . ') '. $mysqli->error;
				/*
				do {
					$error .= ', (' . $mysqli->connect_errno . ') '. $mysqli->connect_error;
				} while ( $mysqli->next_result () );*/
				
				return $this->_trigger_error ( $error, self::ERR_DB_QUERY, NULL, __FILE__, __LINE__ );
			}
			
			// 释放结果集
			do {
				/* store first result set */
				if ($result = $mysqli->store_result ()) {
					$result->close ();
				}
			} while ( $mysqli->next_result () );
		}
		
		$chunk_sqls = array($aReturn5Min);
		foreach ( $chunk_sqls as $sqls ) {
			$query = implode(';', $sqls);
				
			/* execute multi query */
			if (false == $mysqli->multi_query ( $query )) {
				$error = 'Query Error';
				$error .= ' (' . $mysqli->errno . ') '. $mysqli->error;
				return $this->_trigger_error ( $error, self::ERR_DB_QUERY, NULL, __FILE__, __LINE__ );
			}
			
			// 释放结果集
			do {
				/* store first result set */
				if ($result = $mysqli->store_result ()) {
					$result->close ();
				}
			} while ( $mysqli->next_result () );
		}
		
		//$mysqli->close();
		
		$this->_log("End save realtime data");
		
		Debug::mark('end_save_realtime_data');
		
		$this->_log("Cost time " . Debug::useTime('start_save_realtime_data','end_save_realtime_data').'s');
		
		return $aReturn;
	}
	
	protected function _unpack_row($rowdata, $fields){
		$rowpos = 0;
		$rowvalues = array();
		foreach($fields as $field=>$f){
			if(empty($f)){
				continue;
			}
	
			$value = substr($rowdata, $rowpos, $f['length']);
	
			if($f['format']){
				$value = current(unpack($f['format'], $value));
			}
	
			$rowpos += $f['length'];
	
			$rowvalues[] = $value;
		}
	
		return $rowvalues;
	}
	
	protected function _run_field_callback($callback, $file, $line, $column, $field, $value, $options){
		// 直接回调,提高性能
		$func = $callback[1];
		return $this->$func($file, $line, $column, $field, $value, $options);

	}
	
	protected function _run_row_callback($callback, $file, $line, $fieldValues, $rowValues, $options){
		// 直接回调,提高性能
		$func = $callback[1];
		return $this->$func($file, $line, $fieldValues, $rowValues, $options);
	}
	
	public static function autoCharset($string, $from='gbk', $to='utf-8'){
		$from = strtoupper($from) == 'UTF8' ? 'utf-8' : $from;
        $to = strtoupper($to) == 'UTF8' ? 'utf-8' : $to;
        if (strtoupper($from) === strtoupper($to) || empty($string) || (is_scalar($string) && !is_string($string))) {
            //如果编码相同或者非字符串标量则不转换
            return $string;
        }
        if (is_string($string)) {
            if (function_exists('mb_convert_encoding')) {
                return mb_convert_encoding($string, $to, $from);
            } elseif (function_exists('iconv')) {
                return iconv($from, $to, $string);
            } else {
                return $string;
            }
        } elseif (is_array($string)) {
            foreach ($string as $key => $val) {
                $_key = self::autoCharset($key, $from, $to);
                $string[$_key] = self::autoCharset($val, $from, $to);
                if ($key != $_key)
                    unset($string[$key]);
            }
            return $string;
        }
        else {
            return $string;
        }
	}
	
	/*
	 
	 sub subRPT156_CSV {
    local(*mydata);
    my($i,$i2,$str,$j,$buffer,@f,$fstr,$s2,$k);
 
    *mydata = $_[0];$i2=length($mydata);print($i2."\r\n");
    $buf1=' 'x1024000;$s2="时间,代码,名称,price3,vol2,Open_Int,price2,LastClose,Open,High,Low,NewPrice,Volume,Amount,BP1,BP2,BP3,BP4,BP5,BV1,BV2,BV3,BV4,BV5,SP1,SP2,SP3,SP4,SP5,SV1,SV2,SV3,SV4,SV5\r\n";
    $k=0;&subBufferReplace(*buf1,$s2,$k);$k+=length($s2);
    for ($i=0;$i<$i2;$i+=156) {
            $str=substr($mydata,$i,156);
            $m_time=unpack("l",substr($str,0,4));($sec,$min,$hour,$mday,$mon,$year,$wan) = gmtime($m_time+28800);$year=$year+1900; $mon=$mon+1;
            $m_szLabel=substr($str,4,12);$m_szLabel=~s/\x00//g;    $m_szName=substr($str,16,16);$m_szName=~s/\x00//g;
            $fstr="%04d-%02d-%02d %02d:%02d:%02d,%s,%s";
            for ($j=0;$j<31;$j++) {    $f[$j]=unpack("f",substr($str,32+$j*4,4)); $fstr.=",%.3f";};$fstr.="\r\n";
            $s2=sprintf($fstr,$year,$mon,$mday,$hour,$min,$sec,$m_szLabel,$m_szName,$f[0],$f[1],$f[2],$f[3],$f[4],$f[5],$f[6],$f[7],$f[8],$f[9],$f[10],$f[11],$f[12],$f[13],$f[14],$f[15],$f[16],$f[17],$f[18],$f[19],$f[20],$f[21],$f[22],$f[23],$f[24],$f[25],$f[26],$f[27],$f[28],$f[29],$f[30]);
            &subBufferReplace(*buf1,$s2,$k);$k+=length($s2);
    }
    $mydata=substr($buf1,0,$k);
    return 1;
}
	 */
}





没有更多推荐了,返回首页