Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion hololinked/storage/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .database import ThingDB
from .database import ThingDB, MongoThingDB
from .json_storage import ThingJSONStorage
from ..utils import get_a_filename_from_instance

Expand All @@ -9,6 +9,11 @@ def prepare_object_storage(instance, **kwargs):
):
filename = kwargs.get("json_filename", f"{get_a_filename_from_instance(instance, extension='json')}")
instance.db_engine = ThingJSONStorage(filename=filename, instance=instance)
elif kwargs.get(
"use_mongo_db", instance.__class__.use_mongo_db if hasattr(instance.__class__, "use_mongo_db") else False
):
config_file = kwargs.get("db_config_file", None)
instance.db_engine = MongoThingDB(instance=instance, config_file=config_file)
elif kwargs.get(
"use_default_db", instance.__class__.use_default_db if hasattr(instance.__class__, "use_default_db") else False
):
Expand Down
123 changes: 123 additions & 0 deletions hololinked/storage/database.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
import os
import threading
import typing
import base64
from sqlalchemy import create_engine, select, inspect as inspect_database
from sqlalchemy.ext import asyncio as asyncio_ext
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Integer, String, JSON, LargeBinary
from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase, MappedAsDataclass
from sqlite3 import DatabaseError
from pymongo import MongoClient, errors as mongo_errors
from ..param import Parameterized
from ..core.property import Property
from dataclasses import dataclass

from ..param import Parameterized
Expand Down Expand Up @@ -458,5 +462,124 @@ def __exit__(self, exc_type, exc_value, exc_tb) -> None:
except Exception as ex:
pass

class MongoThingDB:
"""
MongoDB-backed database engine for Thing properties and info.

This class provides persistence for Thing properties using MongoDB.
Properties are stored in the 'properties' collection, with fields:
- id: Thing instance identifier
- name: property name
- serialized_value: serialized property value

Methods mirror the interface of ThingDB for compatibility.
"""
def __init__(self, instance: Parameterized, config_file: typing.Union[str, None] = None) -> None:
"""
Initialize MongoThingDB for a Thing instance.
Connects to MongoDB and sets up collections.
"""
self.thing_instance = instance
self.id = instance.id
self.config = self.load_conf(config_file)
self.client = MongoClient(self.config.get("mongo_uri", "mongodb://localhost:27017"))
self.db = self.client[self.config.get("database", "hololinked")]
self.properties = self.db["properties"]
self.things = self.db["things"]

@classmethod
def load_conf(cls, config_file: str) -> typing.Dict[str, typing.Any]:
"""
Load configuration from JSON file if provided.
"""
if not config_file:
return {}
elif config_file.endswith(".json"):
with open(config_file, "r") as file:
return JSONSerializer.load(file)
else:
raise ValueError(f"config files of extension - ['json'] expected, given file name {config_file}")

def fetch_own_info(self):
"""
Fetch Thing instance metadata from the 'things' collection.
"""
doc = self.things.find_one({"id": self.id})
return doc

def get_property(self, property: typing.Union[str, Property], deserialized: bool = True) -> typing.Any:
"""
Get a property value from MongoDB for this Thing.
If deserialized=True, returns the Python value.
"""
name = property if isinstance(property, str) else property.name
doc = self.properties.find_one({"id": self.id, "name": name})
if not doc:
raise mongo_errors.PyMongoError(f"property {name} not found in database")
if not deserialized:
return doc
serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, name)
return serializer.loads(base64.b64decode(doc["serialized_value"]))

def set_property(self, property: typing.Union[str, Property], value: typing.Any) -> None:
"""
Set a property value in MongoDB for this Thing.
Value is serialized before storage.
"""
name = property if isinstance(property, str) else property.name
serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, name)
serialized_value = base64.b64encode(serializer.dumps(value)).decode("utf-8")
self.properties.update_one(
{"id": self.id, "name": name},
{"$set": {"serialized_value": serialized_value}},
upsert=True
)

def get_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any], deserialized: bool = True) -> typing.Dict[str, typing.Any]:
"""
Get multiple property values from MongoDB for this Thing.
Returns a dict of property names to values.
"""
names = [obj if isinstance(obj, str) else obj.name for obj in properties.keys()]
cursor = self.properties.find({"id": self.id, "name": {"$in": names}})
result = {}
for doc in cursor:
serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, doc["name"])
result[doc["name"]] = doc["serialized_value"] if not deserialized else serializer.loads(base64.b64decode(doc["serialized_value"]))
return result

def set_properties(self, properties: typing.Dict[typing.Union[str, Property], typing.Any]) -> None:
"""
Set multiple property values in MongoDB for this Thing.
"""
for obj, value in properties.items():
name = obj if isinstance(obj, str) else obj.name
serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, name)
serialized_value = base64.b64encode(serializer.dumps(value)).decode("utf-8")
self.properties.update_one(
{"id": self.id, "name": name},
{"$set": {"serialized_value": serialized_value}},
upsert=True
)

def get_all_properties(self, deserialized: bool = True) -> typing.Dict[str, typing.Any]:
cursor = self.properties.find({"id": self.id})
result = {}
for doc in cursor:
serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, doc["name"])
result[doc["name"]] = doc["serialized_value"] if not deserialized else serializer.loads(base64.b64decode(doc["serialized_value"]))
return result

def create_missing_properties(self, properties: typing.Dict[str, Property], get_missing_property_names: bool = False) -> typing.Any:
missing_props = []
existing_props = self.get_all_properties()
for name, new_prop in properties.items():
if name not in existing_props:
serializer = Serializers.for_object(self.id, self.thing_instance.__class__.__name__, new_prop.name)
serialized_value = base64.b64encode(serializer.dumps(getattr(self.thing_instance, new_prop.name))).decode("utf-8")
self.properties.insert_one({"id": self.id, "name": new_prop.name, "serialized_value": serialized_value})
missing_props.append(name)
if get_missing_property_names:
return missing_props

__all__ = [BaseAsyncDB.__name__, BaseSyncDB.__name__, ThingDB.__name__, batch_db_commit.__name__]
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ dependencies = [
"jsonschema>=4.22.0,<5.0",
"httpx>=0.28.1,<29.0",
"sniffio>=1.3.1,<2.0",
"pymongo>=4.15.2",
]

[project.urls]
Expand Down
4 changes: 0 additions & 4 deletions tests/test_07_properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,6 @@


class TestProperty(TestCase):
@classmethod
def setUpClass(self):
super().setUpClass()
print(f"test property with {self.__name__}")

def test_01_simple_class_property(self):
"""Test basic class property functionality"""
Expand Down
80 changes: 80 additions & 0 deletions tests/working/test_07_properties_mongodb.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import unittest
from hololinked.core.property import Property
from hololinked.core import Thing
from hololinked.storage.database import MongoThingDB
from pymongo import MongoClient

class TestMongoDBOperations(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Clear MongoDB 'properties' collection before tests
try:
client = MongoClient("mongodb://localhost:27017")
db = client["hololinked"]
db["properties"].delete_many({})
except Exception as e:
print(f"Warning: Could not clear MongoDB test data: {e}")

def test_mongo_string_property(self):
class MongoTestThing(Thing):
str_prop = Property(default="hello", db_persist=True)
instance = MongoTestThing(id="mongo_str", use_mongo_db=True)
instance.str_prop = "world"
value_from_db = instance.db_engine.get_property("str_prop")
self.assertEqual(value_from_db, "world")

def test_mongo_float_property(self):
class MongoTestThing(Thing):
float_prop = Property(default=1.23, db_persist=True)
instance = MongoTestThing(id="mongo_float", use_mongo_db=True)
instance.float_prop = 4.56
value_from_db = instance.db_engine.get_property("float_prop")
self.assertAlmostEqual(value_from_db, 4.56)

def test_mongo_bool_property(self):
class MongoTestThing(Thing):
bool_prop = Property(default=False, db_persist=True)
instance = MongoTestThing(id="mongo_bool", use_mongo_db=True)
instance.bool_prop = True
value_from_db = instance.db_engine.get_property("bool_prop")
self.assertTrue(value_from_db)

def test_mongo_dict_property(self):
class MongoTestThing(Thing):
dict_prop = Property(default={"a": 1}, db_persist=True)
instance = MongoTestThing(id="mongo_dict", use_mongo_db=True)
instance.dict_prop = {"b": 2, "c": 3}
value_from_db = instance.db_engine.get_property("dict_prop")
self.assertEqual(value_from_db, {"b": 2, "c": 3})

def test_mongo_list_property(self):
class MongoTestThing(Thing):
list_prop = Property(default=[1, 2], db_persist=True)
instance = MongoTestThing(id="mongo_list", use_mongo_db=True)
instance.list_prop = [3, 4, 5]
value_from_db = instance.db_engine.get_property("list_prop")
self.assertEqual(value_from_db, [3, 4, 5])

def test_mongo_none_property(self):
class MongoTestThing(Thing):
none_prop = Property(default=None, db_persist=True, allow_None=True)
instance = MongoTestThing(id="mongo_none", use_mongo_db=True)
instance.none_prop = None
value_from_db = instance.db_engine.get_property("none_prop")
self.assertIsNone(value_from_db)

def test_mongo_property_persistence(self):
thing_id = "mongo_test_persistence_unique"
prop_name = "test_prop_unique"
client = MongoClient("mongodb://localhost:27017")
db = client["hololinked"]
db["properties"].delete_many({"id": thing_id, "name": prop_name})
class MongoTestThing(Thing):
test_prop_unique = Property(default=123, db_persist=True)
instance = MongoTestThing(id=thing_id, use_mongo_db=True)
instance.test_prop_unique = 456
value_from_db = instance.db_engine.get_property(prop_name)
self.assertEqual(value_from_db, 456)

if __name__ == "__main__":
unittest.main()
Loading