From 732e62d284bf2286613fee100f6a9780fa45620a Mon Sep 17 00:00:00 2001 From: charlene tau express Date: Mon, 17 Mar 2025 16:02:15 +0800 Subject: [PATCH] arrange .env --- src/.env | 9 +++++++ src/app.py | 4 +-- src/qdrant_services/__init__.py | 0 src/qdrant_services/config.py | 18 +++++++++++++ src/qdrant_services/qdrant.py | 31 ++++++++++++----------- src/qdrant_services/settings.py | 8 ------ src/services/__init__.py | 0 src/services/config.py | 6 +++++ src/services/embedding_service.py | 13 ++++++---- src/util/config.py | 42 ++++++++++++++----------------- 10 files changed, 78 insertions(+), 53 deletions(-) create mode 100644 src/.env create mode 100644 src/qdrant_services/__init__.py create mode 100644 src/qdrant_services/config.py delete mode 100644 src/qdrant_services/settings.py create mode 100644 src/services/__init__.py create mode 100644 src/services/config.py diff --git a/src/.env b/src/.env new file mode 100644 index 0000000..47b0eea --- /dev/null +++ b/src/.env @@ -0,0 +1,9 @@ +QDRANT_HOST = "192.168.99.122" +QDRANT_PORT = 6333 +QDRANT_HTTPS='true' +QDRANT_SSL_VERIFY= 'false' +QDRANT_API_KEY=NS00TXlKUUIweUhWaGFuUUpUVTk6bWNVWXI1VXRSN2VWcFRtaEZ6NmdCUQ== +QDRANT_CLUSTER='' +QDRANT_COLLECTION_NAME='titan-2502110203' +TOKENIZER_FOLDER=/home/charlenewsl/model-files/tokenizer +MODEL_FOLDER=/home/charlenewsl/model-files \ No newline at end of file diff --git a/src/app.py b/src/app.py index 1ffa5ff..dd85f17 100644 --- a/src/app.py +++ b/src/app.py @@ -1,5 +1,5 @@ from flask import Flask, jsonify, request -from dotenv import load_dotenv +# from dotenv import load_dotenv from services.embedding_service import Embedding import logging import json @@ -7,7 +7,7 @@ from qdrant_client.http import models logger = logging.getLogger(__name__) -load_dotenv() +# load_dotenv() app = Flask(__name__) # @app.route("/") diff --git a/src/qdrant_services/__init__.py b/src/qdrant_services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/qdrant_services/config.py b/src/qdrant_services/config.py new file mode 100644 index 0000000..d5b6d05 --- /dev/null +++ b/src/qdrant_services/config.py @@ -0,0 +1,18 @@ +# replace ./settings.py +from util.config import env + +QDRANT_HOST = env('QDRANT_HOST', default="192.168.99.122") +QDRANT_PORT = env('QDRANT_PORT', default=6333, cast=int) +QDRANT_HTTPS = env('QDRANT_HTTPS', default=False, cast=bool) +QDRANT_SSL_VERIFY = env('QDRANT_SSL_VERIFY', default=False, cast=bool) +QDRANT_API_KEY = env('QDRANT_API_KEY', default=None) +QDRANT_CLUSTER = env('QDRANT_CLUSTER', default='') +QDRANT_COLLECTION_NAME=env('QDRANT_COLLECTION_NAME',default='titan-2502110203') + + + + + + +# TOKENIZER_FOLDER=/home/charlenewsl/model-files/tokenizer +# MODEL_FOLDER=/home/charlenewsl/model-files \ No newline at end of file diff --git a/src/qdrant_services/qdrant.py b/src/qdrant_services/qdrant.py index 5fc8d6c..6a78361 100644 --- a/src/qdrant_services/qdrant.py +++ b/src/qdrant_services/qdrant.py @@ -5,7 +5,7 @@ from functools import wraps from qdrant_client import QdrantClient -from qdrant_services.settings import QDRANT_HOST, QDRANT_PORT, QDRANT_HTTPS, QDRANT_SSL_VERIFY,QDRANT_API_KEY, QDRANT_CLUSTER, QDRANT_COLLECTION_NAME +from . import config as settings from services.service import Singleton from qdrant_services.policy import QdrantClientSelectionPolicy @@ -25,16 +25,16 @@ class Qdrant(metaclass=Singleton): ): self.collection_name = collection_name or self.__class__.get_default_collection_name() - if QDRANT_HTTPS is not None and kwargs.get('https') is None: - kwargs['https'] = QDRANT_HTTPS - if QDRANT_SSL_VERIFY is not None and kwargs.get('verify') is None: - kwargs['verify'] = QDRANT_SSL_VERIFY - if QDRANT_API_KEY is not None and kwargs.get('api_key') is None: - kwargs['api_key'] = QDRANT_API_KEY + if settings.QDRANT_HTTPS is not None and kwargs.get('https') is None: + kwargs['https'] = settings.QDRANT_HTTPS + if settings.QDRANT_SSL_VERIFY is not None and kwargs.get('verify') is None: + kwargs['verify'] = settings.QDRANT_SSL_VERIFY + if settings.QDRANT_API_KEY is not None and kwargs.get('api_key') is None: + kwargs['api_key'] = settings.QDRANT_API_KEY - host = host or QDRANT_HOST - port = port or QDRANT_PORT - cluster = cluster or QDRANT_CLUSTER + host = host or settings.QDRANT_HOST + port = port or settings.QDRANT_PORT + cluster = cluster or settings.QDRANT_CLUSTER if cluster: self.logger.info( "Connecting to qdrant cluster: cluster: %s, collection_name: %s" % (cluster, self.collection_name)) @@ -82,16 +82,17 @@ class Qdrant(metaclass=Singleton): @classmethod def get_default_collection_name(cls) -> str: - return QDRANT_COLLECTION_NAME + return settings.QDRANT_COLLECTION_NAME class QdrantClientProxy(QdrantClient): def __init__(self, outer_instance, *args, **kwargs): self.outer_instance = outer_instance - super().__init__(url="https://192.168.99.122:6333", - api_key="NS00TXlKUUIweUhWaGFuUUpUVTk6bWNVWXI1VXRSN2VWcFRtaEZ6NmdCUQ==", - https=True, - verify=False) + super().__init__(*args, **kwargs) + # super().__init__(url="https://192.168.99.122:6333", + # api_key="NS00TXlKUUIweUhWaGFuUUpUVTk6bWNVWXI1VXRSN2VWcFRtaEZ6NmdCUQ==", + # https=True, + # verify=False) _excluded_methods = ['_dynamic_call_decorator'] diff --git a/src/qdrant_services/settings.py b/src/qdrant_services/settings.py deleted file mode 100644 index 48d6653..0000000 --- a/src/qdrant_services/settings.py +++ /dev/null @@ -1,8 +0,0 @@ -QDRANT_HOST = "192.168.99.122" -QDRANT_PORT = 6333 -QDRANT_HTTPS='' -QDRANT_SSL_VERIFY='' -QDRANT_API_KEY='' -QDRANT_CLUSTER='' -QDRANT_COLLECTION_NAME='titan-2502110203' - diff --git a/src/services/__init__.py b/src/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/services/config.py b/src/services/config.py new file mode 100644 index 0000000..b9bd92f --- /dev/null +++ b/src/services/config.py @@ -0,0 +1,6 @@ +# replace ./settings.py +from util.config import env + + +TOKENIZER_FOLDER=env('TOKENIZER_FOLDER') +MODEL_FOLDER = env('MODEL_FOLDER') diff --git a/src/services/embedding_service.py b/src/services/embedding_service.py index ff85014..1c071a8 100644 --- a/src/services/embedding_service.py +++ b/src/services/embedding_service.py @@ -6,10 +6,13 @@ import torch import os from transformers import AutoTokenizer, AutoModel -# from app.settings import MODEL_FOLDER, TOKENIZER_FOLDER + from services.service import Component -TOKENIZER_FOLDER = os.getenv("TOKENIZER_FOLDER") -MODEL_FOLDER = os.getenv("MODEL_FOLDER") + +from . import config as settings + + + class Embedding(Component): def __init__(self): @@ -24,8 +27,8 @@ class Embedding(Component): # Load model from HuggingFace Hub - self.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_FOLDER) - self.model = AutoModel.from_pretrained(MODEL_FOLDER) + self.tokenizer = AutoTokenizer.from_pretrained(settings.TOKENIZER_FOLDER) + self.model = AutoModel.from_pretrained(settings.MODEL_FOLDER) def process(self, input_clause, is_query=False): diff --git a/src/util/config.py b/src/util/config.py index e0bb30a..2ed8555 100644 --- a/src/util/config.py +++ b/src/util/config.py @@ -1,28 +1,24 @@ -import logging +from dotenv import dotenv_values import os -import environ -from services.service import Singleton - -logger = logging.getLogger(__name__) +# load_dotenv("/home/charlenewsl/recommender-service-flask/src/.env") +# print("QDRANT_PORT: DD : ",os.getenv("QDRANT_PORT")) +config = dotenv_values("/home/charlenewsl/recommender-service-flask/src/.env") -def init(): - env = environ.Env(DEBUG=(bool, False)) - path = os.path.join((environ.Path(__file__) - 3).__str__(), '.env') - if os.path.exists(path): - environ.Env.read_env(path) - logger.info("env file %s loaded" % path) - # no warning even if .env doesn't exist as it's not required - # e.g. for docker containers, usually environment variables are preferred. - - return env - - -class Env(metaclass=Singleton): - def __init__(self): - self.env = init() - - -env = Env().env +def env(name, default=None, cast=None): + r = os.getenv(name) + if r is None: + r = config.get(name, default) + if cast is not None: + if cast == bool and isinstance(r, str): + if r.lower() in ["true", "yes", "on", "1"]: + r = True + elif r.lower() in ["false", "no", "off", "0"]: + r = False + else: + raise ValueError(f"Invalid boolean value: {r}") + else: + r = cast(r) + return r