Add S3 functionality #2
1
.gitignore
vendored
1
.gitignore
vendored
@ -181,3 +181,4 @@ tags
|
||||
# Persistent undo
|
||||
[._]*.un~
|
||||
|
||||
apply_environment.sh
|
||||
|
@ -1,18 +1,39 @@
|
||||
from flask_restx import Namespace, Resource, fields
|
||||
from flask_restx import reqparse
|
||||
import s3Client
|
||||
import logging
|
||||
|
||||
# Exported namespace
|
||||
api = Namespace('search', description='Searching for memes')
|
||||
|
||||
query = api.model('Query', {
|
||||
'query': fields.String(required=True, description='Search string', example='doge'),
|
||||
def getClientSafely():
|
||||
logging.debug("Getting a client safely...")
|
||||
client = None
|
||||
try:
|
||||
client = s3Client.getClient()
|
||||
except Exception as e:
|
||||
logging.critical("Failed to retrive a client : " + str(e))
|
||||
|
||||
})
|
||||
return client
|
||||
|
||||
@api.route('/exact')
|
||||
|
||||
@api.route('/exact/<string:query>')
|
||||
@api.doc(params={
|
||||
'query': 'Name of the meme you are looking for'
|
||||
})
|
||||
class exactSearch(Resource):
|
||||
@api.doc('search')
|
||||
def get(self):
|
||||
return {"message" : "hello world!"}
|
||||
@api.response(200, 'Sucess')
|
||||
@api.response(500, 'S3 Error')
|
||||
def get(self, query):
|
||||
logging.debug("Getting a client safely...")
|
||||
client = getClientSafely()
|
||||
if client is None:
|
||||
return {
|
||||
"message": "Error connecting to S3"
|
||||
}, 500
|
||||
if query in client.getCurrentMemeList():
|
||||
return "nice"
|
||||
else:
|
||||
return "boo"
|
||||
|
||||
|
@ -4,6 +4,9 @@
|
||||
from flask import Flask
|
||||
from api import api
|
||||
import os
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
isDebug = True
|
||||
|
||||
|
105
src/s3Client.py
Normal file
105
src/s3Client.py
Normal file
@ -0,0 +1,105 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
|
||||
from minio import Minio
|
||||
from minio.commonconfig import Tags
|
||||
from minio.error import S3Error
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
S3_URL = ""
|
||||
S3_UN = ""
|
||||
S3_PW = ""
|
||||
S3_TLS = True
|
||||
S3_BUCKET = ""
|
||||
|
||||
gclient = None
|
||||
|
||||
def getClient():
|
||||
global gclient
|
||||
global S3_URL
|
||||
global S3_UN
|
||||
global S3_PW
|
||||
global S3_TLS
|
||||
global S3_BUCKET
|
||||
if gclient != None:
|
||||
return gclient
|
||||
|
||||
if "S3_URL" not in os.environ:
|
||||
raise Exception("S3_URL is not set!")
|
||||
S3_URL = os.environ["S3_URL"]
|
||||
logging.info("Using S3_URL : " + S3_URL )
|
||||
|
||||
|
||||
if "S3_UN" not in os.environ:
|
||||
raise Exception("S3_UN is not set!")
|
||||
S3_UN = os.environ["S3_UN"]
|
||||
logging.info("Using S3_UN : " + S3_UN)
|
||||
|
||||
if "S3_PW" not in os.environ:
|
||||
raise Exception("S3_PW is not set!")
|
||||
S3_PW = os.environ["S3_PW"]
|
||||
logging.info("Using S3_PW : " + S3_PW)
|
||||
|
||||
if "S3_BUCKET" not in os.environ:
|
||||
raise Exception("S3_BUCKET is not set!")
|
||||
S3_BUCKET = os.environ["S3_BUCKET"]
|
||||
logging.info("Using S3_BUCKET : " + S3_BUCKET)
|
||||
|
||||
# override defaults
|
||||
if "S3_TLS" in os.environ:
|
||||
S3_TLS = os.environ["S3_TLS"].lower() in ("yes", "true", "1", "t")
|
||||
logging.info("Using S3_TLS : " + str(S3_TLS))
|
||||
|
||||
client = Minio(S3_URL,
|
||||
access_key=S3_UN,
|
||||
secret_key=S3_PW,
|
||||
secure=S3_TLS)
|
||||
found = client.bucket_exists(S3_BUCKET)
|
||||
if not found:
|
||||
client.make_bucket(S3_BUCKET)
|
||||
logging.info(f"Failed to find bucket " + S3_BUCKET + " so I made it instead")
|
||||
else:
|
||||
logging.info(f"Found bucket " + S3_BUCKET)
|
||||
|
||||
clientObject = Client(client)
|
||||
gclient = clientObject
|
||||
|
||||
return gclient
|
||||
|
||||
class Client:
|
||||
allMemes = set()
|
||||
memesToMd5 = dict()
|
||||
lastCheckedAllMemes = datetime.strptime("2000-01-01 01:01:01", "%Y-%m-%d %H:%M:%S")
|
||||
client = None
|
||||
|
||||
def __init__(self, client):
|
||||
global gclient
|
||||
if gclient is not None:
|
||||
raise Exception("Object already exists! use getClient")
|
||||
logging.debug("Making a new client")
|
||||
if client is not None:
|
||||
self.client = client
|
||||
else:
|
||||
raise Exception("Improper object passed for client!")
|
||||
|
||||
def getCurrentMemeList(self, force=False):
|
||||
now = datetime.now()
|
||||
if (now - self.lastCheckedAllMemes).seconds > 300 or force:
|
||||
logging.info("Enough time has elapsed, refreshing meme cache...")
|
||||
self.lastCheckedAllMemes = now
|
||||
|
||||
self.allMemes.clear()
|
||||
self.memesToMd5.clear()
|
||||
|
||||
for obj in self.client.list_objects(S3_BUCKET):
|
||||
if not obj.is_dir:
|
||||
self.allMemes.add(obj.object_name)
|
||||
self.memesToMd5[obj.etag] = obj.object_name
|
||||
logging.info("Finished fetching " + str(len(self.allMemes)) + " memes")
|
||||
|
||||
return self.allMemes
|
||||
|
||||
|
Reference in New Issue
Block a user