{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "### Проверка работы с модулем 25 через API" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Обучение модели и генерация данных" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##### Файловый API" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import requests\n", "import json\n", "from requests.auth import HTTPBasicAuth\n", "import time\n", "import datetime" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "module_name = \"pu-mgalynchik-pa-mm25-synth\"\n", "username = \"developer\"\n", "password = \"WpNJxt7vCdP4Q9TYFX5Sh6ukGZHyrD\"\n", "REQUESTS_TIMEOUT = 20\n", "\n", "basic_auth = HTTPBasicAuth(username, password)\n", "\n", "headers = {\n", " \"Content-Type\": \"application/json\"\n", "}\n", "\n", "service_url = f\"https://platform-dev.stratpro.hse.ru/{module_name}\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# Путь к файлу с входными данными, как локально, так и в файловом хранилище фреймворка\n", "filename = \"data2/telecom/telecom_data.csv\"" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/data2/telecom/telecom_data.csv',\n", " 'presigned_put_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/data2/telecom/telecom_data.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T195650Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=2c8ad0adc28eeda89d2d2274971d6051fd8894a5dd32d1caa55b11b1777948ba'}" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "response = requests.put(f\"{service_url}/files/synthdata-box/{filename}\", auth=basic_auth)\n", "if response.status_code in [400, ]:\n", " response = requests.get(f\"{service_url}/files/synthdata-box/{filename}\", auth=basic_auth)\n", "display(response.json())" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "200\n" ] } ], "source": [ "if response.status_code in [200, 201]:\n", " result_urls = json.loads(response.content) \n", " \n", " with open(filename, \"rb\") as f:\n", " response = requests.put(result_urls[\"presigned_put_url\"], data=f.read())\n", " print(response.status_code)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##### Запуск пайплайна генерации данных" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "pipeline_name = \"train-generate\"\n", "full_url = f\"https://platform-dev.stratpro.hse.ru/{module_name}/pipelines/{pipeline_name}/trials\"" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "synthdata_request = {\n", " \"inputs\": [\n", " {\n", " \"name\": \"input_data\",\n", " \"datatype\": \"FILE\",\n", " \"content_type\": \"text/csv\",\n", " \"shape\": [7043, 20],\n", " \"data\": \"data2/telecom/\"\n", " }\n", " ],\n", " \"output_vars\": [\n", " {\n", " \"name\": \"report_file\",\n", " \"data\": \"results2/syntesize_data/report/\"\n", " },\n", " {\n", " \"name\": \"output_data\",\n", " \"data\": \"results2/syntesize_data/output/\"\n", " },\n", " {\n", " \"name\": \"model\",\n", " \"data\": \"results2/syntesize_data/model/\"\n", " },\n", " ]\n", "}" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'https://platform-dev.stratpro.hse.ru/pu-mgalynchik-pa-mm25-synth/trials/4c4b3d7d-d960-4c1d-ad31-c35b1f5f1744'" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tracking_url" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'app_name': 'pu-mgalynchik-pa-mm25-synth',\n", " 'user_id': 'developer',\n", " 'tracking_id': '4c4b3d7d-d960-4c1d-ad31-c35b1f5f1744',\n", " 'pipeline_name': 'train-generate',\n", " 'pipeline_version': '34929876',\n", " 'next_pipeline_name': None,\n", " 'next_tracking_id': None,\n", " 'vars': {'inputs': [{'name': 'input_data',\n", " 'box_name': 'synthdata-box',\n", " 'data': 'data2/telecom/',\n", " 'datatype': 'FILE',\n", " 'content_type': 'text/csv'}],\n", " 'outputs': [{'name': 'report_file',\n", " 'box_name': 'synthdata-box',\n", " 'data': 'results2/syntesize_data/report/',\n", " 'datatype': 'FILE',\n", " 'content_type': None},\n", " {'name': 'output_data',\n", " 'box_name': 'synthdata-box',\n", " 'data': 'results2/syntesize_data/output/',\n", " 'datatype': 'FILE',\n", " 'content_type': None},\n", " {'name': 'model',\n", " 'box_name': 'synthdata-box',\n", " 'data': 'results2/syntesize_data/model/',\n", " 'datatype': 'FILE',\n", " 'content_type': None}]},\n", " 'status': {'conditions': [{'type': 'Started',\n", " 'condition_status': 'True',\n", " 'last_transition_time': '2024-12-20T19:57:04.310958+0000',\n", " 'message': '',\n", " 'reason': '',\n", " 'stage': None}],\n", " 'status_date_time': '2024-12-20T19:57:04.310958+0000'},\n", " '_links': {'self': {'href': 'https://platform-dev.stratpro.hse.ru/pu-mgalynchik-pa-mm25-synth/trials/4c4b3d7d-d960-4c1d-ad31-c35b1f5f1744',\n", " 'templated': None},\n", " 'pipeline': {'href': 'https://platform-dev.stratpro.hse.ru/pu-mgalynchik-pa-mm25-synth/pipelines/train-generate',\n", " 'templated': None},\n", " 'next_trial': None}}" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "response.json()" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tracking_info" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0:00:00] Pipeline job status - Started\n", "[0:02:29] Pipeline job status - OutputsAreValidated\n", "[0:02:32] Pipeline job status - Completed\n" ] }, { "data": { "text/plain": [ "{'type': 'Completed',\n", " 'condition_status': 'True',\n", " 'last_transition_time': '2024-12-20T19:59:36.835439+0000',\n", " 'message': '',\n", " 'reason': '',\n", " 'stage': None}" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "tracking_info = requests.get(tracking_url, auth=basic_auth)\n", "last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n", "status_history = [last_status]\n", "while last_status['type'] != 'Completed':\n", " if last_status not in status_history:\n", " status_history.append(last_status)\n", " \n", " time.sleep(REQUESTS_TIMEOUT)\n", " \n", " tracking_info = requests.get(tracking_url, auth=basic_auth)\n", " last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n", "\n", "status_conditions = json.loads(tracking_info.content)['status']['conditions']\n", "start_time = datetime.datetime.strptime(status_conditions[0]['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n", "for idx, status in enumerate(status_conditions):\n", " status_time = datetime.datetime.strptime(status['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n", " print(f\"[{str(status_time - start_time).split('.', maxsplit=1)[0]}] Pipeline job status - {status['type']}\")\n", "\n", "display(last_status)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "ename": "JSONDecodeError", "evalue": "Expecting value: line 1 column 1 (char 0)", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[8], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;241m200\u001b[39m, ]:\n\u001b[1;32m 5\u001b[0m tracking_info \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(tracking_url, auth\u001b[38;5;241m=\u001b[39mbasic_auth)\n\u001b[0;32m----> 6\u001b[0m last_status \u001b[38;5;241m=\u001b[39m \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtracking_info\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcontent\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstatus\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mconditions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m 7\u001b[0m status_history \u001b[38;5;241m=\u001b[39m [last_status]\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m last_status[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCompleted\u001b[39m\u001b[38;5;124m'\u001b[39m:\n", "File \u001b[0;32m/usr/lib/python3.10/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 341\u001b[0m s \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mdecode(detect_encoding(s), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msurrogatepass\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 348\u001b[0m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n", "File \u001b[0;32m/usr/lib/python3.10/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, s, _w\u001b[38;5;241m=\u001b[39mWHITESPACE\u001b[38;5;241m.\u001b[39mmatch):\n\u001b[1;32m 333\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[38;5;124;03m containing a JSON document).\u001b[39;00m\n\u001b[1;32m 335\u001b[0m \n\u001b[1;32m 336\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_w\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mend\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 338\u001b[0m end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m 339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n", "File \u001b[0;32m/usr/lib/python3.10/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m 353\u001b[0m obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscan_once(s, idx)\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpecting value\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, err\u001b[38;5;241m.\u001b[39mvalue) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 356\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj, end\n", "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)" ] } ], "source": [ "response = requests.post(full_url, headers=headers, auth=basic_auth, json=synthdata_request)\n", "tracking_url = json.loads(response.content)['_links']['self']['href']\n", "\n", "if response.status_code in [200, ]:\n", " tracking_info = requests.get(tracking_url, auth=basic_auth)\n", " last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n", " status_history = [last_status]\n", " while last_status['type'] != 'Completed':\n", " if last_status not in status_history:\n", " status_history.append(last_status)\n", " \n", " time.sleep(REQUESTS_TIMEOUT)\n", " \n", " tracking_info = requests.get(tracking_url, auth=basic_auth)\n", " last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n", "\n", " status_conditions = json.loads(tracking_info.content)['status']['conditions']\n", " start_time = datetime.datetime.strptime(status_conditions[0]['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n", " for idx, status in enumerate(status_conditions):\n", " status_time = datetime.datetime.strptime(status['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n", " print(f\"[{str(status_time - start_time).split('.', maxsplit=1)[0]}] Pipeline job status - {status['type']}\")\n", " \n", " display(last_status)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##### Результаты" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/report/',\n", " 'files': [{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/report/report.json',\n", " 'presigned_get_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/results2/syntesize_data/report/report.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T200128Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=6d3f3dfae9d69d7b73f82876dfde6626c5448851722c1deac0abdfd7e12a6105'}]}" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/output/',\n", " 'files': [{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/output/prediction.csv',\n", " 'presigned_get_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/results2/syntesize_data/output/prediction.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T200128Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=c70d7ebbd679cc2ed83390911d8578237459d38f78ed4b2d05ef951f8b1a1c20'}]}" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/model/',\n", " 'files': [{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/model/model.joblib',\n", " 'presigned_get_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/results2/syntesize_data/model/model.joblib?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T200128Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=98993f950c6295a422dde2b2f46ed6c996bd565c55c8f90777bf48c9001094d6'}]}" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "for filename in synthdata_request[\"output_vars\"]:\n", " response = requests.get(f\"{service_url}/files/synthdata-box/{filename['data']}\", auth=basic_auth)\n", " if response.status_code in [200, 201]:\n", " display(response.json())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##### Процент совпадений с входными данными:" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Overall Quality Score': 0.9492168339013363}" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "report_fname = synthdata_request[\"output_vars\"][0]['data']\n", "response = requests.get(f\"{service_url}/files/synthdata-box/{report_fname}\", auth=basic_auth)\n", "if response.status_code in [200, 201]:\n", " response = requests.get(response.json()['files'][0]['presigned_get_url'], auth=basic_auth)\n", " display(response.json())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 2 }