From 6ce46e39e07cb0d47bf3bdd935ad72c996ea6e47 Mon Sep 17 00:00:00 2001 From: Nya Date: Thu, 29 Oct 2020 22:32:29 +0100 Subject: [PATCH] #1 Prepare data --- .gitignore | 3 +++ README.md | 24 +++++++++++++++++ .../data/storybook-learning-events.csv | 26 +++++++++++++++++++ env-TEST/lang-ENG/data/storybooks.csv | 7 +++++ prepare_data.py | 26 +++++++++++++++++++ 5 files changed, 86 insertions(+) create mode 100644 env-TEST/lang-ENG/data/storybook-learning-events.csv create mode 100644 env-TEST/lang-ENG/data/storybooks.csv create mode 100644 prepare_data.py diff --git a/.gitignore b/.gitignore index b6e4761..bd6ad26 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# PyCharm +.idea/ diff --git a/README.md b/README.md index 2cbf805..c3a9a8b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,26 @@ # ml-storybooks-recommender + Machine learning model which predicts the rating of unread storybooks based on the student's previously read storybooks. + +One model will be trained per language. + + +## 1. Prepare the Data + +To prepare the data, follow these steps: + * Open `prepare_data.py` and select environment and language. + * Go to the website corresponding to the chosen environment and language, e.g. http://eng.test.elimu.ai. + * Download `storybooks.csv` from http://eng.test.elimu.ai/content/storybook/list. + * Download `storybook-learning-events.csv` from http://eng.test.elimu.ai/analytics/storybook-learning-event/list. + * Add the two datasets to `RAW_DATA_DIR`. + * Execute the script: `python prepare_data.py` + + +## 2. Train the Model + +TODO + + +## 3. Make Predictions on New Samples + +TODO diff --git a/env-TEST/lang-ENG/data/storybook-learning-events.csv b/env-TEST/lang-ENG/data/storybook-learning-events.csv new file mode 100644 index 0000000..1156fec --- /dev/null +++ b/env-TEST/lang-ENG/data/storybook-learning-events.csv @@ -0,0 +1,26 @@ +id,time,android_id,package_name,storybook_id,learning_event_type +45,1603495877976,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +44,1603495780823,7394f9d23bfd74af,ai.elimu.vitabu.test,24,STORYBOOK_OPENED +43,1603495757361,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +42,1603494702681,7394f9d23bfd74af,ai.elimu.vitabu.test,23,STORYBOOK_OPENED +41,1603494681753,7394f9d23bfd74af,ai.elimu.vitabu.test,24,STORYBOOK_OPENED +40,1603494581648,7394f9d23bfd74af,ai.elimu.vitabu.test,24,STORYBOOK_OPENED +39,1603494430877,7394f9d23bfd74af,ai.elimu.vitabu.test,27,STORYBOOK_OPENED +38,1603494363802,7394f9d23bfd74af,ai.elimu.vitabu.test,27,STORYBOOK_OPENED +37,1603494110370,7394f9d23bfd74af,ai.elimu.vitabu.test,26,STORYBOOK_OPENED +36,1603493867772,7394f9d23bfd74af,ai.elimu.vitabu.test,25,STORYBOOK_OPENED +35,1603493388681,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +34,1603493323420,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +33,1603493261035,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +32,1603493235566,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +31,1603493014112,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +30,1603492973830,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +29,1603492930084,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +28,1603492919979,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +27,1603492918728,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +26,1603492908493,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +25,1603492907090,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +24,1603492903577,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +23,1603492807897,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +22,1603492757816,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED +21,1603492731620,7394f9d23bfd74af,ai.elimu.vitabu.test,22,STORYBOOK_OPENED diff --git a/env-TEST/lang-ENG/data/storybooks.csv b/env-TEST/lang-ENG/data/storybooks.csv new file mode 100644 index 0000000..00f3c31 --- /dev/null +++ b/env-TEST/lang-ENG/data/storybooks.csv @@ -0,0 +1,7 @@ +id,title,description,content_license,attribution_url,reading_level,cover_image_id,chapters +27,Grace in Space,Girl travels with her uncle to space and returns years later.,CREATIVE_COMMONS_CC_BY_NC_SA,https://digitallibrary.io/en/books/details/1349,LEVEL2,114,"[{""image"":{""id"":116},""sortOrder"":1,""id"":297,""storyBookParagraphs"":[{""originalText"":""Fifth grade student, Little Miss Grace,"",""sortOrder"":0,""id"":535},{""originalText"":""was totally fascinated by outer space."",""sortOrder"":1,""id"":536}]},{""image"":{""id"":117},""sortOrder"":2,""id"":298,""storyBookParagraphs"":[{""originalText"":""Astronomy books, she loved to read,"",""sortOrder"":0,""id"":537},{""originalText"":""and she knew all the stars and"",""sortOrder"":1,""id"":538},{""originalText"":""galaxies indeed."",""sortOrder"":2,""id"":539}]},{""image"":{""id"":118},""sortOrder"":3,""id"":299,""storyBookParagraphs"":[{""originalText"":""She spent wonderful times with her uncle, Mr. Scope, observing the planets with her new telescope."",""sortOrder"":0,""id"":540}]},{""image"":{""id"":119},""sortOrder"":4,""id"":300,""storyBookParagraphs"":[{""originalText"":""She was so happy to find out that he made a"",""sortOrder"":0,""id"":541},{""originalText"":""spaceship, and that he\u2019ll take her with him on a faraway trip."",""sortOrder"":1,""id"":542}]},{""image"":{""id"":120},""sortOrder"":5,""id"":301,""storyBookParagraphs"":[{""originalText"":""He said: \u201cGet ready, get set, we\u2019re about to go!"",""sortOrder"":0,""id"":543},{""originalText"":""To visit Kepler, the Earth-like planet you know!\u201d"",""sortOrder"":1,""id"":544}]},{""image"":{""id"":121},""sortOrder"":6,""id"":302,""storyBookParagraphs"":[{""originalText"":""\u201cFor my spaceship travels as fast as light!"",""sortOrder"":0,""id"":545},{""originalText"":""And we\u2019ll have lots of fun during our flight.\u201d"",""sortOrder"":1,""id"":546}]},{""image"":{""id"":122},""sortOrder"":7,""id"":303,""storyBookParagraphs"":[{""originalText"":""She put on her spacesuit and waved goodbye."",""sortOrder"":0,""id"":547},{""originalText"":""Then, three, two, one. Blast off to the sky!"",""sortOrder"":1,""id"":548}]},{""image"":{""id"":123},""sortOrder"":8,""id"":304,""storyBookParagraphs"":[{""originalText"":""Her uncle cheered, \u201cWe\u2019re off to the stars!"",""sortOrder"":0,""id"":549},{""originalText"":""Off to Kepler far beyond Venus and Mars.\u201d"",""sortOrder"":1,""id"":550}]},{""image"":{""id"":124},""sortOrder"":9,""id"":305,""storyBookParagraphs"":[{""originalText"":""When the spaceship landed on Kepler\u2019s surface,"",""sortOrder"":0,""id"":551},{""originalText"":""Grace was amazed by all its beauty and fairness."",""sortOrder"":1,""id"":552}]},{""image"":{""id"":125},""sortOrder"":10,""id"":306,""storyBookParagraphs"":[{""originalText"":""She asked her uncle where the aliens could be."",""sortOrder"":0,""id"":553},{""originalText"":""He said, \u201cYou can watch them on a sci-fi movie.\u201d"",""sortOrder"":1,""id"":554},{""originalText"":""\u201cWe don\u2019t have lots of time to spend around here. So, let\u2019s collect some samples and souvenirs.\u201d"",""sortOrder"":2,""id"":555}]},{""image"":{""id"":126},""sortOrder"":11,""id"":307,""storyBookParagraphs"":[{""originalText"":""On their way back, they noticed something strange."",""sortOrder"":0,""id"":556},{""originalText"":""Their city looked so different and their mobiles were out of range!"",""sortOrder"":1,""id"":557}]},{""image"":{""id"":127},""sortOrder"":12,""id"":308,""storyBookParagraphs"":[{""originalText"":""Her uncle put his hand on his head and said, \u201cOh dear, Oh dearie!"",""sortOrder"":0,""id"":558},{""originalText"":""I forgot all about an important theory! Not to mention, the fourth dimension!"",""sortOrder"":1,""id"":559},{""originalText"":""We thought we travelled for hours, but we spent years and years!"",""sortOrder"":2,""id"":560},{""originalText"":""I\u2019m sure our family must have shed lots of tears.\u201d"",""sortOrder"":3,""id"":561}]},{""image"":{""id"":128},""sortOrder"":13,""id"":309,""storyBookParagraphs"":[{""originalText"":""Grace was happy to see her parents that were now very old."",""sortOrder"":0,""id"":562},{""originalText"":""And her baby sister was also a college graduate, she was told."",""sortOrder"":1,""id"":563}]},{""image"":{""id"":129},""sortOrder"":14,""id"":310,""storyBookParagraphs"":[{""originalText"":""Still, being back with her family felt really great,"",""sortOrder"":0,""id"":564},{""originalText"":""And on this happy occasion, they all did celebrate."",""sortOrder"":1,""id"":565}]},{""image"":{""id"":130},""sortOrder"":15,""id"":311,""storyBookParagraphs"":[{""originalText"":""Little Miss Grace in astonishment sung,"",""sortOrder"":0,""id"":566},{""originalText"":""\u201cMy little sister grew older, but I stayed young.\u201d"",""sortOrder"":1,""id"":567}]},{""image"":{""id"":131},""sortOrder"":16,""id"":312,""storyBookParagraphs"":[{""originalText"":""The End"",""sortOrder"":0,""id"":568}]}]" +25,Lara the Yellow Ladybird,"A yellow bug in a world of red, Lara the Ladybird just wants to fit in.",CREATIVE_COMMONS_CC_BY,https://storyweaver.org.in/stories/13916-lara-the-yellow-ladybird,LEVEL1,79,"[{""image"":{""id"":81},""sortOrder"":0,""id"":261,""storyBookParagraphs"":[{""originalText"":""Lara the ladybird was a special bug."",""sortOrder"":0,""id"":484}]},{""image"":{""id"":82},""sortOrder"":1,""id"":262,""storyBookParagraphs"":[{""originalText"":""Unlike all her friends, she had bright yellow wings."",""sortOrder"":0,""id"":485}]},{""image"":{""id"":83},""sortOrder"":2,""id"":263,""storyBookParagraphs"":[{""originalText"":""Everyone loved her yellow wings."",""sortOrder"":0,""id"":486}]},{""image"":{""id"":84},""sortOrder"":3,""id"":264,""storyBookParagraphs"":[{""originalText"":""Each morning, Bibi Butterfly said hello."",""sortOrder"":0,""id"":487},{""originalText"":""And Manto Mantis always waved."",""sortOrder"":1,""id"":488}]},{""image"":{""id"":85},""sortOrder"":4,""id"":265,""storyBookParagraphs"":[{""originalText"":""Even Sesa, the sulky spider,"",""sortOrder"":0,""id"":489},{""originalText"":""was happy to see her."",""sortOrder"":1,""id"":490}]},{""image"":{""id"":86},""sortOrder"":5,""id"":266,""storyBookParagraphs"":[{""originalText"":""At school, she played with lots of friends."",""sortOrder"":0,""id"":491}]},{""image"":{""id"":87},""sortOrder"":6,""id"":267,""storyBookParagraphs"":[{""originalText"":""But Lara wanted"",""sortOrder"":0,""id"":492},{""originalText"":""to be like the"",""sortOrder"":1,""id"":493},{""originalText"":""other ladybirds."",""sortOrder"":2,""id"":494},{""originalText"":""\u201cI wish I had"",""sortOrder"":3,""id"":495},{""originalText"":""red wings like you,"",""sortOrder"":4,""id"":496},{""originalText"":""Mama,\u201d she cried."",""sortOrder"":5,""id"":497}]},{""image"":{""id"":88},""sortOrder"":7,""id"":268,""storyBookParagraphs"":[{""originalText"":""So one day, to cheer her up, Lara\u2019s mother painted her wings bright red."",""sortOrder"":0,""id"":498}]},{""image"":{""id"":89},""sortOrder"":8,""id"":269,""storyBookParagraphs"":[{""originalText"":""The next morning, nobody greeted Lara on her way to school."",""sortOrder"":0,""id"":499}]},{""image"":{""id"":90},""sortOrder"":9,""id"":270,""storyBookParagraphs"":[{""originalText"":""And when she got there, none of her friends said hello."",""sortOrder"":0,""id"":500}]},{""image"":{""id"":91},""sortOrder"":10,""id"":271,""storyBookParagraphs"":[{""originalText"":""Lara sat all alone."",""sortOrder"":0,""id"":501},{""originalText"":""No one noticed"",""sortOrder"":1,""id"":502},{""originalText"":""her new red wings."",""sortOrder"":2,""id"":503}]},{""image"":{""id"":92},""sortOrder"":11,""id"":272,""storyBookParagraphs"":[{""originalText"":""Until Miss Miya spotted her and said: \u201cYou\u2019ve painted your lovely yellow wings!\u201d"",""sortOrder"":0,""id"":504}]},{""image"":{""id"":93},""sortOrder"":12,""id"":273,""storyBookParagraphs"":[{""originalText"":""Lara\u2019s classmates were shocked."",""sortOrder"":0,""id"":505},{""originalText"":""\u201cYour wings are special!\u201d"",""sortOrder"":1,""id"":506},{""originalText"":""\u201cSo unique!\u201d"",""sortOrder"":2,""id"":507},{""originalText"":""\u201cSo rare!\u201d"",""sortOrder"":3,""id"":508}]},{""image"":{""id"":94},""sortOrder"":13,""id"":274,""storyBookParagraphs"":[{""originalText"":""\u201cLara,\u201d Miss Miya said, \u201cyour yellow wings are what make you you."",""sortOrder"":0,""id"":509},{""originalText"":""Like Sipho\u2019s spot\u2026 and Sally\u2019s legs.\u201d"",""sortOrder"":1,""id"":510}]},{""image"":{""id"":95},""sortOrder"":14,""id"":275,""storyBookParagraphs"":[{""originalText"":""Back home, Lara took a long bath and scrubbed until her golden wings gleamed."",""sortOrder"":0,""id"":511}]},{""image"":{""id"":96},""sortOrder"":15,""id"":276,""storyBookParagraphs"":[{""originalText"":""\u201cI\u2019ll never paint my wings again!\u201d"",""sortOrder"":0,""id"":512},{""originalText"":""she thought. Except, maybe once or twice\u2026"",""sortOrder"":1,""id"":513}]},{""image"":{""id"":97},""sortOrder"":16,""id"":277,""storyBookParagraphs"":[{""originalText"":""To try a bit of purple\u2026 or something nice."",""sortOrder"":0,""id"":514},{""originalText"":""But not for ever and just for fun."",""sortOrder"":1,""id"":515}]},{""image"":{""id"":98},""sortOrder"":17,""id"":278,""storyBookParagraphs"":[{""originalText"":""..."",""sortOrder"":0,""id"":516}]},{""image"":{""id"":99},""sortOrder"":18,""id"":279,""storyBookParagraphs"":[{""originalText"":""..."",""sortOrder"":0,""id"":517}]}]" +24,Shoecat Thoocat,"Kamala’s new shoecat attacks whoever it meets! Tomatoes, thathas, friends, teachers, goats… everyone hates it. What will they do now?",CREATIVE_COMMONS_CC_BY,https://storyweaver.org.in/stories/47980-shoecat-thoocat,LEVEL3,58,"[{""image"":{""id"":60},""sortOrder"":0,""id"":239,""storyBookParagraphs"":[{""originalText"":""Once upon a time, a traveller fell from the sky."",""sortOrder"":0,""id"":448}]},{""image"":{""id"":61},""sortOrder"":1,""id"":240,""storyBookParagraphs"":[{""originalText"":""It landed on a field of tomatoes."",""sortOrder"":0,""id"":449},{""originalText"":""Kamala and her Thatha ran out. All their tomatoes were crushed, leaving only a shoe."",""sortOrder"":1,""id"":450}]},{""image"":{""id"":62},""sortOrder"":2,""id"":241,""storyBookParagraphs"":[{""originalText"":""Kamala picked up the shoe. It put out a claw and scratched her. HISSSSS."",""sortOrder"":0,""id"":451},{""originalText"":""\u201cShoo cat!\u201d Kamala yelled."",""sortOrder"":1,""id"":452},{""originalText"":""The shoecat yawned."",""sortOrder"":2,""id"":453}]},{""image"":{""id"":63},""sortOrder"":3,""id"":242,""storyBookParagraphs"":[{""originalText"":""Kamala picked up the shoecat and took it to the goat shed. She gave it a bowl of milk. The shoecat sniffed it and spat in disgust. Thoo! Then it ate up all the goats\u2019 grass."",""sortOrder"":0,""id"":454},{""originalText"":""After that, the shoecat went wherever Kamala went."",""sortOrder"":1,""id"":455}]},{""image"":{""id"":64},""sortOrder"":4,""id"":243,""storyBookParagraphs"":[{""originalText"":""At school, it scratched Shamim Miss. \u201cShoo! Cat!\u201d she yelled at it. The shoecat spat at her. "",""sortOrder"":0,""id"":456},{""originalText"":""It scratched all Kamala\u2019s classmates. Where the shoecat scratched, the wounds glowed green."",""sortOrder"":1,""id"":457}]},{""image"":{""id"":65},""sortOrder"":5,""id"":244,""storyBookParagraphs"":[{""originalText"":""The shoecat scratched the newspaper man. It scratched seventeen goats, thirteen hens, three dogs, and one Thatha. It scratched everyone except Kamala, because it had already scratched her."",""sortOrder"":0,""id"":458},{""originalText"":""\u201cOw! Shoo cat!\u201d they all yelled in Kannada and English, Tamil and Urdu and Telugu, Goatish and Doggo and Chickenam. The cat replied, \u201cThooo!\u201d"",""sortOrder"":1,""id"":459}]},{""image"":{""id"":66},""sortOrder"":6,""id"":245,""storyBookParagraphs"":[{""originalText"":""The school put up a sign: NO CATS"",""sortOrder"":0,""id"":460}]},{""image"":{""id"":67},""sortOrder"":7,""id"":246,""storyBookParagraphs"":[{""originalText"":""The shop put up a sign, too: CATS NOT ALLOWED"",""sortOrder"":0,""id"":461}]},{""image"":{""id"":68},""sortOrder"":8,""id"":247,""storyBookParagraphs"":[{""originalText"":""Even the playground got a sign: CATS STAY OUT"",""sortOrder"":0,""id"":462}]},{""image"":{""id"":69},""sortOrder"":9,""id"":248,""storyBookParagraphs"":[{""originalText"":""Kamala and the shoecat went home, sadly. "",""sortOrder"":0,""id"":463},{""originalText"":""They found Thatha putting up his own sign: CAT GO BACK"",""sortOrder"":1,""id"":464}]},{""image"":{""id"":70},""sortOrder"":10,""id"":249,""storyBookParagraphs"":[{""originalText"":""That night, Kamala and the shoecat climbed up to the roof to watch the skies. The moon glowed green and warm. "",""sortOrder"":0,""id"":465},{""originalText"":""The shoecat turned its head up and yowled, \u201cOwoooo!\u201d"",""sortOrder"":1,""id"":466}]},{""image"":{""id"":71},""sortOrder"":11,""id"":250,""storyBookParagraphs"":[{""originalText"":""In the goat shed, seventeen goats turned their heads up and howled: Owoooo!"",""sortOrder"":0,""id"":467},{""originalText"":""In his bedroom, Thatha sat up and put on his glasses. Sleepily, he said, \u201cOwoo?\u201d"",""sortOrder"":1,""id"":468}]},{""image"":{""id"":72},""sortOrder"":12,""id"":251,""storyBookParagraphs"":[{""originalText"":""Across the village, Kamala\u2019s teacher, her classmates, the newspaper man, thirteen hens, and three dogs lifted their faces to the green moon and howled: \u201cOwoooo!\u201d"",""sortOrder"":0,""id"":469},{""originalText"":""On the roof, Kamala bellowed: \n\n\u201cOWOOO!\u201d\n\n"",""sortOrder"":1,""id"":470}]},{""image"":{""id"":73},""sortOrder"":13,""id"":252,""storyBookParagraphs"":[{""originalText"":"" "",""sortOrder"":0,""id"":471},{""originalText"":""With a flash of lightning, it began to rain. The no-cat signs melted away."",""sortOrder"":1,""id"":472}]},{""image"":{""id"":74},""sortOrder"":14,""id"":253,""storyBookParagraphs"":[{""originalText"":""Then the thunder started, BA-DUM DUM-DUM! Every piece of glass in the village broke. PUTT! went the light bulbs. Pop-POP went the pickle jars. Mirrors CRACKED and crumbled. "",""sortOrder"":0,""id"":473},{""originalText"":""Windows slammed against walls and smashed glass flew everywhere, even on the roof. "",""sortOrder"":1,""id"":474}]},{""image"":{""id"":75},""sortOrder"":15,""id"":254,""storyBookParagraphs"":[{""originalText"":""Kamala hid under her bed."",""sortOrder"":0,""id"":475}]},{""image"":{""id"":76},""sortOrder"":16,""id"":255,""storyBookParagraphs"":[{""originalText"":""The next morning, Kamala crawled out from under her bed. Thatha was sitting on the steps. He was eating a furry green tomato."",""sortOrder"":0,""id"":476},{""originalText"":""\u201cWonderful! Thank your thoocat,\u201d Thatha said. \u201cWhat a good shoecat!\u201d"",""sortOrder"":1,""id"":477}]},{""image"":{""id"":77},""sortOrder"":17,""id"":256,""storyBookParagraphs"":[{""originalText"":""In the field were lots of tall tomato plants, some medium-sized goats, and one short shoecat."",""sortOrder"":0,""id"":478}]},{""image"":{""id"":78},""sortOrder"":18,""id"":257,""storyBookParagraphs"":[{""originalText"":""\u201cThoo!\u201d said the shoecat, in a friendly way."",""sortOrder"":0,""id"":479},{""originalText"":""\u201cThoo!\u201d Kamala agreed as she bit into a green tomato."",""sortOrder"":1,""id"":480}]}]" +23,The Candy Planet,Monsters going to a candy planet for a family trip.,CREATIVE_COMMONS_CC_BY,https://storyweaver.org.in/stories/118416-the-candy-planet,LEVEL3,45,"[{""image"":{""id"":47},""sortOrder"":0,""id"":223,""storyBookParagraphs"":[{""originalText"":""\""Hooray!\"" shouted Spooky the monster.  We are finally going to Planet Candy for a family trip! \""Yeah!\"" exclaimed Spurk, \""We can eat as many candies as we want!\"" They danced around the living room.  Afterwards, they called their dad. \""Dad!\"" they yelled, \""Yes?\"" Dad replied.  The boys said together, \""Are we starting to pack up for Planet Candy yet? \""Yes!\"" Dad replied.  So, they started packing."",""sortOrder"":0,""id"":431}]},{""image"":{""id"":48},""sortOrder"":1,""id"":224,""storyBookParagraphs"":[{""originalText"":""They packed their T-shirts, pants, toys, books, video games, paper, pen, etc. After a while, they finally finished packing. They double check their bags again to make sure they had everything they needed for Planet Candy. Afterwards, they climbed aboard for the Ultra Shuttle and waited for the shuttle to lift off."",""sortOrder"":0,""id"":432}]},{""image"":{""id"":49},""sortOrder"":2,""id"":225,""storyBookParagraphs"":[{""originalText"":""After a few minutes, mom asked the boys, \""Are you ready for take off?!\""  \""You bet!\"" Spooky and Spurk replied happily.  Everybody excitedly counted down together \""10,9,8,7,6,5,4,3,2,1...blast off!  \""Wheeeeeeeeee! the boys shouted.  After a few hours, they finally arrived at Planet Candy.  \""Prepare for landing.\"" dad said."",""sortOrder"":0,""id"":433}]},{""image"":{""id"":50},""sortOrder"":3,""id"":226,""storyBookParagraphs"":[{""originalText"":""Spooky shouted, \""WOW! the clouds are made of cotton candy!\""  Mom said, \""Yup! that is why it is called the candy planet!\"" Finally, they landed on Planet Candy.  Then, they took a bus to their hotel.  \""COOL! exclaimed Spooky.  \""That bridge is made out of candy canes!  Soon, they arrived at their hotel.  The other guest monsters were polite, and the hotel was beautiful."",""sortOrder"":0,""id"":434}]},{""image"":{""id"":51},""sortOrder"":4,""id"":227,""storyBookParagraphs"":[{""originalText"":""Later on, it was dinner time.  Suddenly, an enormous monster that was way bigger than the other monsters arose.  It had big hands, sharp teeth, and a gigantic body.  Everybody were shaking with fear, they were drowning in sweat, their hands shaking and their eyes closed. "",""sortOrder"":0,""id"":435}]},{""image"":{""id"":52},""sortOrder"":5,""id"":228,""storyBookParagraphs"":[{""originalText"":""Out of nowhere, a spider came out from the toilet.  But this spider was not the normal spider.  It was a spider that had special powers.  The powers were: enormous breath, silk lasso and shooting fire balls from it's butt."",""sortOrder"":0,""id"":436}]},{""image"":{""id"":53},""sortOrder"":6,""id"":229,""storyBookParagraphs"":[{""originalText"":""Then, the battle began.  First, the spider shot fire balls from it's butt.  Next, he splashed out tons of silk lasso and tied them around the big monster.  The big monster could not move at all, and he tried so hard to escape but failed.  The big monster was furious!"",""sortOrder"":0,""id"":437}]},{""image"":{""id"":54},""sortOrder"":7,""id"":230,""storyBookParagraphs"":[{""originalText"":""Next, working together all the small monsters invented the largest trash can in the whole world! \""That was magnificent!\"" Spooky and his family said together.  So, they all lifted the big monster into the gigantic trash can and after that, they flushed the trash can down the toilet.  Everybody cheered.  But the real hero was the spider.  He saved the day!"",""sortOrder"":0,""id"":438},{""originalText"":""."",""sortOrder"":1,""id"":439}]},{""image"":{""id"":55},""sortOrder"":8,""id"":231,""storyBookParagraphs"":[{""originalText"":"" Afterwards, everybody celebrated and had candies for dinner.  They were eating bubble gums out of flower petals, marshmallows from flowers, hi-chews from trees, lolly pops from twigs.  They could even travel to the sky and reached cotton candies out of the clouds.  Everybody enjoyed eating so many candies."",""sortOrder"":0,""id"":440}]},{""image"":{""id"":56},""sortOrder"":9,""id"":232,""storyBookParagraphs"":[{""originalText"":""Spooky and Spurk exclaimed, \""This was the best family trip ever!\"" \""Of course! We would never ever let you two eat that many candies again!  Just only for this time!\"" mom and dad concluded.  Spooky and Spurk both announced together, \"" We would want to come back here next year again! But do you think the big monster will arise again?\"""",""sortOrder"":0,""id"":441}]},{""image"":{""id"":57},""sortOrder"":10,""id"":233,""storyBookParagraphs"":[{""originalText"":""THE END"",""sortOrder"":0,""id"":442}]}]" +26,Today I Am,"What will Megha be today? An explorer, an astronaut, an artist?",CREATIVE_COMMONS_CC_BY,https://storyweaver.org.in/stories/43643-today-i-am,LEVEL1,100,"[{""image"":{""id"":102},""sortOrder"":0,""id"":284,""storyBookParagraphs"":[{""originalText"":""Today, I am an astronaut."",""sortOrder"":0,""id"":521}]},{""image"":{""id"":103},""sortOrder"":1,""id"":285,""storyBookParagraphs"":[{""originalText"":""Look how high I can"",""sortOrder"":0,""id"":522},{""originalText"":""jump on the moon."",""sortOrder"":1,""id"":523}]},{""image"":{""id"":104},""sortOrder"":2,""id"":286,""storyBookParagraphs"":[{""originalText"":""Today, I am a sculptor."",""sortOrder"":0,""id"":524}]},{""image"":{""id"":105},""sortOrder"":3,""id"":287,""storyBookParagraphs"":[{""originalText"":""Watch out, my crocodile and dinosaur are hungry. Rawrrr!"",""sortOrder"":0,""id"":525}]},{""image"":{""id"":106},""sortOrder"":4,""id"":288,""storyBookParagraphs"":[{""originalText"":""Today, I am a cricketer."",""sortOrder"":0,""id"":526}]},{""image"":{""id"":107},""sortOrder"":5,""id"":289,""storyBookParagraphs"":[{""originalText"":""It is my turn to bowl. And\u2026 you are OUT!"",""sortOrder"":0,""id"":527}]},{""image"":{""id"":108},""sortOrder"":6,""id"":290,""storyBookParagraphs"":[{""originalText"":""Today, I am a botanist."",""sortOrder"":0,""id"":528}]},{""image"":{""id"":109},""sortOrder"":7,""id"":291,""storyBookParagraphs"":[{""originalText"":""I\u2019ve discovered a new plant. I\u2019ll name it after myself."",""sortOrder"":0,""id"":529}]},{""image"":{""id"":110},""sortOrder"":8,""id"":292,""storyBookParagraphs"":[{""originalText"":""Today, I am a drummer."",""sortOrder"":0,""id"":530}]},{""image"":{""id"":111},""sortOrder"":9,""id"":293,""storyBookParagraphs"":[{""originalText"":""Dhushh! Tsshh! My sticks are moving so fast, you cannot even see them."",""sortOrder"":0,""id"":531}]},{""image"":{""id"":112},""sortOrder"":10,""id"":294,""storyBookParagraphs"":[{""originalText"":""Today, I am so many things."",""sortOrder"":0,""id"":532},{""originalText"":""Tomorrow, what will I be?"",""sortOrder"":1,""id"":533}]}]" +22,Wild Cat! Wild Cat!,Meet the different wild cats of India.,CREATIVE_COMMONS_CC_BY,https://digitallibrary.io/en/books/details/1855,LEVEL1,29,"[{""image"":{""id"":31},""sortOrder"":0,""id"":208,""storyBookParagraphs"":[{""originalText"":""Some wild cats have stripes."",""sortOrder"":0,""id"":393},{""originalText"":""Like the tiger!"",""sortOrder"":1,""id"":394}]},{""image"":{""id"":32},""sortOrder"":1,""id"":209,""storyBookParagraphs"":[{""originalText"":""Some wild cats have manes."",""sortOrder"":0,""id"":395},{""originalText"":""Like the lion!"",""sortOrder"":1,""id"":396}]},{""image"":{""id"":33},""sortOrder"":2,""id"":210,""storyBookParagraphs"":[{""originalText"":""Some wild cats have spots."",""sortOrder"":0,""id"":397},{""originalText"":""Like the leopard and the leopard cat!"",""sortOrder"":1,""id"":398}]},{""image"":{""id"":34},""sortOrder"":3,""id"":211,""storyBookParagraphs"":[{""originalText"":""Some wild cats are the best climbers."",""sortOrder"":0,""id"":399},{""originalText"":""Like the clouded leopard and the marbled cat!"",""sortOrder"":1,""id"":400}]},{""image"":{""id"":35},""sortOrder"":4,""id"":212,""storyBookParagraphs"":[{""originalText"":""Some wild cats live high up in the mountains."",""sortOrder"":0,""id"":401},{""originalText"":""Like the snow leopard and the Palla's cat!"",""sortOrder"":1,""id"":402}]},{""image"":{""id"":36},""sortOrder"":5,""id"":213,""storyBookParagraphs"":[{""originalText"":""Some wild cats live in the desert."",""sortOrder"":0,""id"":403},{""originalText"":""Like the desert cat!"",""sortOrder"":1,""id"":404}]},{""image"":{""id"":37},""sortOrder"":6,""id"":214,""storyBookParagraphs"":[{""originalText"":""Some wild cats have hairy ears."",""sortOrder"":0,""id"":405},{""originalText"":""Like the caracal and the lynx!"",""sortOrder"":1,""id"":406}]},{""image"":{""id"":38},""sortOrder"":7,""id"":215,""storyBookParagraphs"":[{""originalText"":""Some wild cats love to fish."",""sortOrder"":0,""id"":407},{""originalText"":""Like the fishing cat!"",""sortOrder"":1,""id"":408}]},{""image"":{""id"":39},""sortOrder"":8,""id"":216,""storyBookParagraphs"":[{""originalText"":""Some wild cats are the smallest wild cats in the world."",""sortOrder"":0,""id"":409},{""originalText"":""Like the rusty-spotted cat!"",""sortOrder"":1,""id"":410}]},{""image"":{""id"":40},""sortOrder"":9,""id"":217,""storyBookParagraphs"":[{""originalText"":""Some wild cats look like house cats."",""sortOrder"":0,""id"":411},{""originalText"":""Like the jungle cat and the golden cat!"",""sortOrder"":1,""id"":412}]},{""image"":{""id"":41},""sortOrder"":10,""id"":218,""storyBookParagraphs"":[{""originalText"":""But they are all wild cats!"",""sortOrder"":0,""id"":413}]}]" diff --git a/prepare_data.py b/prepare_data.py new file mode 100644 index 0000000..950452c --- /dev/null +++ b/prepare_data.py @@ -0,0 +1,26 @@ +# This Python script prepares the following two datasets: +# 1. Storybooks +# 2. Storybook learning events + +import pandas as pd + +# Select environment (TEST/PROD) +ENVIRONMENT = "TEST" + +# Select language +# See https://github.com/elimu-ai/model/blob/master/src/main/java/ai/elimu/model/enums/Language.java +LANGUAGE = "ENG" + +RAW_DATA_DIR = "./env-" + ENVIRONMENT + "/lang-" + LANGUAGE + "/data" +print(f"RAW_DATA_DIR: {RAW_DATA_DIR}") + +# Load the storybooks +storybooks_pd = pd.read_csv(RAW_DATA_DIR + "/storybooks.csv") +print(f"storybooks_pd: \n{storybooks_pd}") + +# Load the storybook learning events +learning_events_pd = pd.read_csv(RAW_DATA_DIR + "/storybook-learning-events.csv") +print(f"learning_events_pd: \n{learning_events_pd}") + +# Clean the data +# TODO