csv导mongo

import logging
import os
import time

import pandas as pd

from ..mongo_ import MongoL

logger = logging.getLogger(__name__)


class CsvMongo(object):

    def __init__(self):
        self.file_list = []
        mongo = MongoL()
        self.mongo_cli, self.collection = mongo.mongo_col()

    def csv_file(self, dir_path):
        for root, dirs, files in os.walk(dir_path):
            for file in files:
                if ".csv" in file:
                    self.file_list.append(file)

        return self.file_list

    def write_db(self, file, app_id, module_id):
        read_ = pd.read_csv(file, chunksize=10000, usecols=['url', 'username'])
        for dg in read_:
            for _, line in dg.iterrows():
                url = line[0]
                username = line[1]
                data = {"app_id": app_id, "module_id": module_id, "result": username,
                        "result_url": url, "app_type_key": "community", "origin": {},
                        "ct": int(time.time())}
                res = self.collection.update_one({"app_id": 4, "module_id": 4, "result": data.get("result")},
                                                 {"$set": data},
                                                 upsert=True)
                logger.info("updata_count:%s,instering:%s" % (res.matched_count, data))

    # def find_

    def csv_mongo(self, files):
        for file in files:
            if file == "dgtle.csv":
                self.write_db(file=file, app_id=4, module_id=4)
            elif file == "oschina.csv":
                self.write_db(file=file, app_id=18, module_id=18)
            elif file == "feng.csv":
                self.write_db(file=file, app_id=17, module_id=17)
            elif file == "oppo.csv":
                self.write_db(file=file, app_id=8, module_id=8)
            elif file == "idol001.csv":
                self.write_db(file=file, app_id=19, module_id=19)
            elif file == "gelonghui.csv":
                self.write_db(file=file, app_id=20, module_id=20)


if __name__ == '__main__':
    cm = CsvMongo()
    files = cm.csv_file(os.path.dirname(__file__))
    cm.csv_mongo(files)