Add S3 functionality #2
1
.gitignore
vendored
1
.gitignore
vendored
@ -181,3 +181,4 @@ tags
|
|||||||
# Persistent undo
|
# Persistent undo
|
||||||
[._]*.un~
|
[._]*.un~
|
||||||
|
|
||||||
|
apply_environment.sh
|
||||||
|
@ -1,18 +1,39 @@
|
|||||||
from flask_restx import Namespace, Resource, fields
|
from flask_restx import Namespace, Resource, fields
|
||||||
|
from flask_restx import reqparse
|
||||||
|
import s3Client
|
||||||
|
import logging
|
||||||
|
|
||||||
|
# Exported namespace
|
||||||
api = Namespace('search', description='Searching for memes')
|
api = Namespace('search', description='Searching for memes')
|
||||||
|
|
||||||
query = api.model('Query', {
|
def getClientSafely():
|
||||||
'query': fields.String(required=True, description='Search string', example='doge'),
|
logging.debug("Getting a client safely...")
|
||||||
|
client = None
|
||||||
|
try:
|
||||||
|
client = s3Client.getClient()
|
||||||
|
except Exception as e:
|
||||||
|
logging.critical("Failed to retrive a client : " + str(e))
|
||||||
|
|
||||||
})
|
return client
|
||||||
|
|
||||||
@api.route('/exact')
|
|
||||||
|
@api.route('/exact/<string:query>')
|
||||||
@api.doc(params={
|
@api.doc(params={
|
||||||
'query': 'Name of the meme you are looking for'
|
'query': 'Name of the meme you are looking for'
|
||||||
})
|
})
|
||||||
class exactSearch(Resource):
|
class exactSearch(Resource):
|
||||||
@api.doc('search')
|
@api.doc('search')
|
||||||
def get(self):
|
@api.response(200, 'Sucess')
|
||||||
return {"message" : "hello world!"}
|
@api.response(500, 'S3 Error')
|
||||||
|
def get(self, query):
|
||||||
|
logging.debug("Getting a client safely...")
|
||||||
|
client = getClientSafely()
|
||||||
|
if client is None:
|
||||||
|
return {
|
||||||
|
"message": "Error connecting to S3"
|
||||||
|
}, 500
|
||||||
|
if query in client.getCurrentMemeList():
|
||||||
|
return "nice"
|
||||||
|
else:
|
||||||
|
return "boo"
|
||||||
|
|
||||||
|
@ -4,6 +4,9 @@
|
|||||||
from flask import Flask
|
from flask import Flask
|
||||||
from api import api
|
from api import api
|
||||||
import os
|
import os
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
isDebug = True
|
isDebug = True
|
||||||
|
|
||||||
|
105
src/s3Client.py
Normal file
105
src/s3Client.py
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
from minio.commonconfig import Tags
|
||||||
|
from minio.error import S3Error
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
S3_URL = ""
|
||||||
|
S3_UN = ""
|
||||||
|
S3_PW = ""
|
||||||
|
S3_TLS = True
|
||||||
|
S3_BUCKET = ""
|
||||||
|
|
||||||
|
gclient = None
|
||||||
|
|
||||||
|
def getClient():
|
||||||
|
global gclient
|
||||||
|
global S3_URL
|
||||||
|
global S3_UN
|
||||||
|
global S3_PW
|
||||||
|
global S3_TLS
|
||||||
|
global S3_BUCKET
|
||||||
|
if gclient != None:
|
||||||
|
return gclient
|
||||||
|
|
||||||
|
if "S3_URL" not in os.environ:
|
||||||
|
raise Exception("S3_URL is not set!")
|
||||||
|
S3_URL = os.environ["S3_URL"]
|
||||||
|
logging.info("Using S3_URL : " + S3_URL )
|
||||||
|
|
||||||
|
|
||||||
|
if "S3_UN" not in os.environ:
|
||||||
|
raise Exception("S3_UN is not set!")
|
||||||
|
S3_UN = os.environ["S3_UN"]
|
||||||
|
logging.info("Using S3_UN : " + S3_UN)
|
||||||
|
|
||||||
|
if "S3_PW" not in os.environ:
|
||||||
|
raise Exception("S3_PW is not set!")
|
||||||
|
S3_PW = os.environ["S3_PW"]
|
||||||
|
logging.info("Using S3_PW : " + S3_PW)
|
||||||
|
|
||||||
|
if "S3_BUCKET" not in os.environ:
|
||||||
|
raise Exception("S3_BUCKET is not set!")
|
||||||
|
S3_BUCKET = os.environ["S3_BUCKET"]
|
||||||
|
logging.info("Using S3_BUCKET : " + S3_BUCKET)
|
||||||
|
|
||||||
|
# override defaults
|
||||||
|
if "S3_TLS" in os.environ:
|
||||||
|
S3_TLS = os.environ["S3_TLS"].lower() in ("yes", "true", "1", "t")
|
||||||
|
logging.info("Using S3_TLS : " + str(S3_TLS))
|
||||||
|
|
||||||
|
client = Minio(S3_URL,
|
||||||
|
access_key=S3_UN,
|
||||||
|
secret_key=S3_PW,
|
||||||
|
secure=S3_TLS)
|
||||||
|
found = client.bucket_exists(S3_BUCKET)
|
||||||
|
if not found:
|
||||||
|
client.make_bucket(S3_BUCKET)
|
||||||
|
logging.info(f"Failed to find bucket " + S3_BUCKET + " so I made it instead")
|
||||||
|
else:
|
||||||
|
logging.info(f"Found bucket " + S3_BUCKET)
|
||||||
|
|
||||||
|
clientObject = Client(client)
|
||||||
|
gclient = clientObject
|
||||||
|
|
||||||
|
return gclient
|
||||||
|
|
||||||
|
class Client:
|
||||||
|
allMemes = set()
|
||||||
|
memesToMd5 = dict()
|
||||||
|
lastCheckedAllMemes = datetime.strptime("2000-01-01 01:01:01", "%Y-%m-%d %H:%M:%S")
|
||||||
|
client = None
|
||||||
|
|
||||||
|
def __init__(self, client):
|
||||||
|
global gclient
|
||||||
|
if gclient is not None:
|
||||||
|
raise Exception("Object already exists! use getClient")
|
||||||
|
logging.debug("Making a new client")
|
||||||
|
if client is not None:
|
||||||
|
self.client = client
|
||||||
|
else:
|
||||||
|
raise Exception("Improper object passed for client!")
|
||||||
|
|
||||||
|
def getCurrentMemeList(self, force=False):
|
||||||
|
now = datetime.now()
|
||||||
|
if (now - self.lastCheckedAllMemes).seconds > 300 or force:
|
||||||
|
logging.info("Enough time has elapsed, refreshing meme cache...")
|
||||||
|
self.lastCheckedAllMemes = now
|
||||||
|
|
||||||
|
self.allMemes.clear()
|
||||||
|
self.memesToMd5.clear()
|
||||||
|
|
||||||
|
for obj in self.client.list_objects(S3_BUCKET):
|
||||||
|
if not obj.is_dir:
|
||||||
|
self.allMemes.add(obj.object_name)
|
||||||
|
self.memesToMd5[obj.etag] = obj.object_name
|
||||||
|
logging.info("Finished fetching " + str(len(self.allMemes)) + " memes")
|
||||||
|
|
||||||
|
return self.allMemes
|
||||||
|
|
||||||
|
|
Reference in New Issue
Block a user