Mongodb快取爬蟲資料
阿新 • • 發佈:2018-11-14
from pymongo import MongoClient from datetime import datetime,timedelta import zlib import pickle class MongoCache: def __init__(self,client=None,expires=timedelta(days=30)): self.client=MongoClient('localhost',27017) self.db=client['cache'] #新建資料庫 '''Mongodb不需要建立表,會自動建立''' self.db.webpage.create_index('timestamp', expireAfterSeconds=expires.total_seconds()) def __getitem__(self, item): record=self.db.webpage.find_one({'_id':item}) if record: return pickle.loads(zlib.decompress(record['result'])) else: raise KeyError(item+' dose not exist!') def __setitem__(self, key, value): record={'result':zlib.compress(pickle.dumps(value)),'timestamp':datetime.utcnow()} self.db.webpage.update({'_id':key},{'$set':record},upset=True)