3 files changed, 30 insertions, 40 deletions
diff --git a/python/api_quotes/__main__.py b/python/api_quotes/__main__.py
index abe65fda..ec926e8e 100644
--- a/python/api_quotes/__main__.py
+++ b/python/api_quotes/__main__.py
@@ -11,14 +11,13 @@ if __name__ == "__main__":
     for i in range(1000):
         if data := MarkitAPI.get_data(asset_class, after):
             for row in data:
-                if row["confidence"] == 10:
-                    try:
-                        quote = Quote.from_markit_line(row)
-                    except ValueError as e:
-                        logger.error(f"Couldn't pase {row['quoteid']}: {e}")
-                    else:
-                        quote.stage()
-                        quote.commit()
+                try:
+                    quote = Quote.from_markit_line(row)
+                except ValueError as e:
+                    logger.error(f"Couldn't pase {row['quoteid']}: {e}")
+                else:
+                    quote.stage()
+                    quote.commit()
             after = f"{row['receiveddatetime']},{asset_class}-9480-{row['quoteid']}"
         else:
             break
diff --git a/python/api_quotes/api.py b/python/api_quotes/api.py
index 60fce5e6..dfcff2df 100644
--- a/python/api_quotes/api.py
+++ b/python/api_quotes/api.py
@@ -18,6 +18,10 @@ def load_api_key():
     return base_url, r.text
 
 
+def lowercase_keys(d):
+    return {k.lower(): v for k, v in d.items()}
+
+
 class MarkitAPI:
     base_url, api_key = load_api_key()
 
@@ -37,13 +41,4 @@ class MarkitAPI:
         path = posixpath.join("parsing", "Quote", service)
         url = urljoin(cls.base_url, path)
         r = requests.get(url, params)
-        print(params)
-        return cls.read_api(r)
-
-    @staticmethod
-    def read_api(r):
-        df = pd.DataFrame.from_dict(json.loads(r.text))
-        if df.empty:
-            return
-        df.columns = df.columns.str.lower()
-        return df.to_dict(orient="records")
+        return map(lowercase_keys, json.loads(r.text))
diff --git a/python/api_quotes/quotes.py b/python/api_quotes/quotes.py
index c1d36817..6b747093 100644
--- a/python/api_quotes/quotes.py
+++ b/python/api_quotes/quotes.py
@@ -15,37 +15,33 @@ def maturity_dt(d):
         return datetime.date(
             int(d["maturityyear"]), int(d["maturitymonth"]), int(d["maturityday"])
         )
-    except ValueError:  # Sometimes maturity isn't included but we still have tenor
+    except (
+        ValueError,
+        KeyError,
+    ):  # Sometimes maturity isn't included but we still have tenor
         return
 
 
 @dataclass
-class QuoteDetails(Deal, table_name="markit_quote_details", deal_type=None):
-    quotetime: datetime
-    quote_source: str
-    id: int
-    sender: str = None
-
-
-@dataclass
 class Quote(Deal, table_name="markit_quotes", deal_type=None):
     quoteid: int
     assetclass: asset_class
-    redcode: str
-    ticker: str
-    maturity: datetime.date
-    tenor: int
-    runningcoupon: int
-    bidconventionalspread: float
-    bidupfront: float
-    bidsize: float
-    askconventionalspread: float
-    askupfront: float
-    asksize: float
-    firmness: firmness
     msg_id: str
-    quotedate: datetime.datetime
     quotesource: str
+    confidence: int
+    redcode: str = None
+    ticker: str = None
+    maturity: datetime.date = None
+    tenor: int = None
+    runningcoupon: int = None
+    bidconventionalspread: float = None
+    bidupfront: float = None
+    bidsize: float = None
+    askconventionalspread: float = None
+    askupfront: float = None
+    asksize: float = None
+    firmness: firmness = None
+    quotedate: datetime.datetime = None
 
     @classmethod
     def from_markit_line(cls, d):