From 270859c504331b6b03d53db636823fd67e59aca4 Mon Sep 17 00:00:00 2001 From: David Luevano <55825613+luevano@users.noreply.github.com> Date: Fri, 11 Oct 2019 23:38:27 -0600 Subject: Rename files --- tirante/__main__.py | 2 +- tirante/ch_man.py | 55 --------------------- tirante/chapter_images_manager.py | 55 +++++++++++++++++++++ tirante/chapters_manager.py | 55 +++++++++++++++++++++ tirante/chim_man.py | 55 --------------------- tirante/cr_dat.py | 87 -------------------------------- tirante/create_database.py | 87 ++++++++++++++++++++++++++++++++ tirante/do_man.py | 50 ------------------- tirante/down_man.py | 6 +-- tirante/download_manager.py | 50 +++++++++++++++++++ tirante/gcil.py | 57 --------------------- tirante/gcl.py | 87 -------------------------------- tirante/get_chapter_image_list.py | 57 +++++++++++++++++++++ tirante/get_chapters_list.py | 87 ++++++++++++++++++++++++++++++++ tirante/up_dat.py | 101 -------------------------------------- tirante/update_database.py | 101 ++++++++++++++++++++++++++++++++++++++ 16 files changed, 496 insertions(+), 496 deletions(-) delete mode 100644 tirante/ch_man.py create mode 100644 tirante/chapter_images_manager.py create mode 100644 tirante/chapters_manager.py delete mode 100644 tirante/chim_man.py delete mode 100644 tirante/cr_dat.py create mode 100644 tirante/create_database.py delete mode 100644 tirante/do_man.py create mode 100644 tirante/download_manager.py delete mode 100644 tirante/gcil.py delete mode 100644 tirante/gcl.py create mode 100644 tirante/get_chapter_image_list.py create mode 100644 tirante/get_chapters_list.py delete mode 100644 tirante/up_dat.py create mode 100644 tirante/update_database.py diff --git a/tirante/__main__.py b/tirante/__main__.py index 3a639c6..60cda28 100644 --- a/tirante/__main__.py +++ b/tirante/__main__.py @@ -20,7 +20,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from cr_dat import create_database +from create_database import create_database # Main manga source. MAIN_URL = 'https://manganelo.com/manga/' diff --git a/tirante/ch_man.py b/tirante/ch_man.py deleted file mode 100644 index ee39248..0000000 --- a/tirante/ch_man.py +++ /dev/null @@ -1,55 +0,0 @@ -"""MIT License - -Copyright (c) 2019 David Luevano - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - - -def chapters_list_to_csv(chapters_list, - manga_name): - """ - Creates a csv file from the input chapter_list. - chapters_list: List of data of the chapters. - manga_name: Name of the manga, folder naming friendly. - """ - - # Adding '.csv' for csv creation. - m_name_ext = ''.join([manga_name, '.csv']) - # print(m_name) - - with open(m_name_ext, 'w') as outcsv: - for chapter in chapters_list: - outcsv.write(''.join([chapter[0], ',', chapter[1], '\n'])) - - -def chapters_csv_to_list(chapter_csv): - """ - Gives a list of chaptesrs from a csv file. - chapters_list: List of data of the chapters. - """ - - out_chapters_list = [] - - with open(chapter_csv, 'r') as incsv: - lines = incsv.readlines() - for line in lines: - out_chapters_list.append(line.strip().split(',')) - - return out_chapters_list diff --git a/tirante/chapter_images_manager.py b/tirante/chapter_images_manager.py new file mode 100644 index 0000000..c589ca5 --- /dev/null +++ b/tirante/chapter_images_manager.py @@ -0,0 +1,55 @@ +"""MIT License + +Copyright (c) 2019 David Luevano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" +from get_chapter_image_list import get_chapter_image_list + + +def chapter_images_list_to_csv(chapter_data): + """ + Creates csv file for a chapter, given the list. + chapter_data: A list containing a url and a title. + """ + + ch_name = ''.join([chapter_data[1], '.csv']) + + chapter_image_list = get_chapter_image_list(chapter_data) + + with open(ch_name, 'w') as outcsv: + for image in chapter_image_list: + outcsv.write(''.join([image[0], ',', image[1], '\n'])) + + +def chapter_images_csv_to_list(chapter_image_csv): + """ + Returns a list given the csv file. + chapter_image_csv: csv containing data for the chapter. + """ + + out_chapter_image_list = [] + + with open(chapter_image_csv, 'r') as incsv: + lines = incsv.readlines() + for line in lines: + # print(line.strip().split(',')) + out_chapter_image_list.append(line.strip().split(',')) + + return out_chapter_image_list diff --git a/tirante/chapters_manager.py b/tirante/chapters_manager.py new file mode 100644 index 0000000..ee39248 --- /dev/null +++ b/tirante/chapters_manager.py @@ -0,0 +1,55 @@ +"""MIT License + +Copyright (c) 2019 David Luevano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + + +def chapters_list_to_csv(chapters_list, + manga_name): + """ + Creates a csv file from the input chapter_list. + chapters_list: List of data of the chapters. + manga_name: Name of the manga, folder naming friendly. + """ + + # Adding '.csv' for csv creation. + m_name_ext = ''.join([manga_name, '.csv']) + # print(m_name) + + with open(m_name_ext, 'w') as outcsv: + for chapter in chapters_list: + outcsv.write(''.join([chapter[0], ',', chapter[1], '\n'])) + + +def chapters_csv_to_list(chapter_csv): + """ + Gives a list of chaptesrs from a csv file. + chapters_list: List of data of the chapters. + """ + + out_chapters_list = [] + + with open(chapter_csv, 'r') as incsv: + lines = incsv.readlines() + for line in lines: + out_chapters_list.append(line.strip().split(',')) + + return out_chapters_list diff --git a/tirante/chim_man.py b/tirante/chim_man.py deleted file mode 100644 index bafb368..0000000 --- a/tirante/chim_man.py +++ /dev/null @@ -1,55 +0,0 @@ -"""MIT License - -Copyright (c) 2019 David Luevano - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" -from gcil import get_chapter_image_list - - -def chapter_image_list_to_csv(chapter_data): - """ - Creates csv file for a chapter, given the list. - chapter_data: A list containing a url and a title. - """ - - ch_name = ''.join([chapter_data[1], '.csv']) - - chapter_image_list = get_chapter_image_list(chapter_data) - - with open(ch_name, 'w') as outcsv: - for image in chapter_image_list: - outcsv.write(''.join([image[0], ',', image[1], '\n'])) - - -def chapter_image_csv_to_list(chapter_image_csv): - """ - Returns a list given the csv file. - chapter_image_csv: csv containing data for the chapter. - """ - - out_chapter_image_list = [] - - with open(chapter_image_csv, 'r') as incsv: - lines = incsv.readlines() - for line in lines: - # print(line.strip().split(',')) - out_chapter_image_list.append(line.strip().split(',')) - - return out_chapter_image_list diff --git a/tirante/cr_dat.py b/tirante/cr_dat.py deleted file mode 100644 index 37a98af..0000000 --- a/tirante/cr_dat.py +++ /dev/null @@ -1,87 +0,0 @@ -"""MIT License - -Copyright (c) 2019 David Luevano - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" -import os - -# Project specific imports. -from gcl import get_chapters_list -from ch_man import chapters_list_to_csv -from ch_man import chapters_csv_to_list -from chim_man import chapter_image_list_to_csv - - -def create_database(main_url, - manga_name_url, - manga_name, - manga_dir, - manga_data_dir): - """ - Creates a database from zero, made of csv files. - main_url: Main webpage name (source). - manga_name_url: Name of the manga in the url format - that's used by the webpage. - manga_name: Actual name of the manga, as it appears in the webpage. - manga_dir: Main manga folder in computer, subfolders here will be created. - manga_data_dir: Main manga data folder in computer. - NOTE: This does not updates the database. - If a database already exists, omits the creation of new files. - """ - - # A better "naming" for the manga, for use with folder creation. - # As well as the name of the main database. - m_name = '_'.join(word.lower() for word in manga_name.split()) - m_name_ext = ''.join([m_name, '.csv']) - - # Navigate to where the main data folder is, - # then to where the manga folder is. - os.chdir(manga_data_dir) - try: - os.mkdir(m_name) - os.chdir(m_name) - except FileExistsError: - print(''.join([m_name, - ' folder already exists.'])) - os.chdir(m_name) - - # List of files and folders in the current path. - data_list_dir = os.listdir() - - # Get the list of chapters, if this already exists, - # read it from the database. - # This is the main manga data. - if m_name_ext not in data_list_dir: - chapters_list = get_chapters_list(main_url=main_url, - manga_name_url=manga_name_url, - manga_name=manga_name) - chapters_list_to_csv(chapters_list=chapters_list, manga_name=m_name) - else: - print(''.join([m_name_ext, ' already exists.'])) - chapters_list = chapters_csv_to_list(m_name_ext) - - # Data for each chapter. - for chapter in chapters_list: - # Get the list for the images of each chapter. - chapter_name_ext = ''.join([chapter[1], '.csv']) - if chapter_name_ext not in data_list_dir: - chapter_image_list_to_csv(chapter) - else: - print(''.join([chapter_name_ext, ' already exists.'])) diff --git a/tirante/create_database.py b/tirante/create_database.py new file mode 100644 index 0000000..20a91df --- /dev/null +++ b/tirante/create_database.py @@ -0,0 +1,87 @@ +"""MIT License + +Copyright (c) 2019 David Luevano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" +import os + +# Project specific imports. +from get_chapters_list import get_chapters_list +from chapters_manager import chapters_list_to_csv +from chapters_manager import chapters_csv_to_list +from chapter_images_manager import chapter_images_list_to_csv + + +def create_database(main_url, + manga_name_url, + manga_name, + manga_dir, + manga_data_dir): + """ + Creates a database from zero, made of csv files. + main_url: Main webpage name (source). + manga_name_url: Name of the manga in the url format + that's used by the webpage. + manga_name: Actual name of the manga, as it appears in the webpage. + manga_dir: Main manga folder in computer, subfolders here will be created. + manga_data_dir: Main manga data folder in computer. + NOTE: This does not updates the database. + If a database already exists, omits the creation of new files. + """ + + # A better "naming" for the manga, for use with folder creation. + # As well as the name of the main database. + m_name = '_'.join(word.lower() for word in manga_name.split()) + m_name_ext = ''.join([m_name, '.csv']) + + # Navigate to where the main data folder is, + # then to where the manga folder is. + os.chdir(manga_data_dir) + try: + os.mkdir(m_name) + os.chdir(m_name) + except FileExistsError: + print(''.join([m_name, + ' folder already exists.'])) + os.chdir(m_name) + + # List of files and folders in the current path. + data_list_dir = os.listdir() + + # Get the list of chapters, if this already exists, + # read it from the database. + # This is the main manga data. + if m_name_ext not in data_list_dir: + chapters_list = get_chapters_list(main_url=main_url, + manga_name_url=manga_name_url, + manga_name=manga_name) + chapters_list_to_csv(chapters_list=chapters_list, manga_name=m_name) + else: + print(''.join([m_name_ext, ' already exists.'])) + chapters_list = chapters_csv_to_list(m_name_ext) + + # Data for each chapter. + for chapter in chapters_list: + # Get the list for the images of each chapter. + chapter_name_ext = ''.join([chapter[1], '.csv']) + if chapter_name_ext not in data_list_dir: + chapter_images_list_to_csv(chapter) + else: + print(''.join([chapter_name_ext, ' already exists.'])) diff --git a/tirante/do_man.py b/tirante/do_man.py deleted file mode 100644 index c72e62f..0000000 --- a/tirante/do_man.py +++ /dev/null @@ -1,50 +0,0 @@ -"""MIT License - -Copyright (c) 2019 David Luevano - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" -import requests - - -def download_image(image_list): - """ - Downloads an image from the specified url, - and saves it with the specified name. - image_list: list that contains url and name. - """ - - # Gets the content of an image rom its url. - img_data = requests.get(image_list[0]).content - - # Opens a file with its corresponding name as 'wb' (write, binary), - # and then, writes the img_data. - with open(image_list[1], 'wb') as handler: - handler.write(img_data) - - -def download_chapter(image_list): - """ - Downloads the whole chapter as images. - image_url_list: List containing urls and file names for each image. - """ - - for image in image_list: - print(image) - download_image(image) diff --git a/tirante/down_man.py b/tirante/down_man.py index 171a5f2..2c17617 100644 --- a/tirante/down_man.py +++ b/tirante/down_man.py @@ -23,9 +23,9 @@ SOFTWARE. import os # Project specific imports. -from ch_man import chapters_csv_to_list -from chim_man import chapter_image_csv_to_list -from do_man import download_chapter +from chapters_manager import chapters_csv_to_list +from chapter_images_manager import chapter_image_csv_to_list +from download_manager import download_chapter def download_manga(manga_name, diff --git a/tirante/download_manager.py b/tirante/download_manager.py new file mode 100644 index 0000000..c72e62f --- /dev/null +++ b/tirante/download_manager.py @@ -0,0 +1,50 @@ +"""MIT License + +Copyright (c) 2019 David Luevano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" +import requests + + +def download_image(image_list): + """ + Downloads an image from the specified url, + and saves it with the specified name. + image_list: list that contains url and name. + """ + + # Gets the content of an image rom its url. + img_data = requests.get(image_list[0]).content + + # Opens a file with its corresponding name as 'wb' (write, binary), + # and then, writes the img_data. + with open(image_list[1], 'wb') as handler: + handler.write(img_data) + + +def download_chapter(image_list): + """ + Downloads the whole chapter as images. + image_url_list: List containing urls and file names for each image. + """ + + for image in image_list: + print(image) + download_image(image) diff --git a/tirante/gcil.py b/tirante/gcil.py deleted file mode 100644 index dbe4572..0000000 --- a/tirante/gcil.py +++ /dev/null @@ -1,57 +0,0 @@ -"""MIT License - -Copyright (c) 2019 David Luevano - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" -import urllib3 -from bs4 import BeautifulSoup - - -def get_chapter_image_list(chapter_data): - """ - Gets the links for each image in the chapter, - and returns a list of the links. - Returns a list of the image urls and its file name. - chapter_data: A list containing a url and a title. - NOTE: Not for direct use with the result of 'get_chapters_list' - """ - - # Not actually a file, but the content of the html. - html = urllib3.PoolManager().request('GET', chapter_data[0]) - - # Get the data from the html and parse it. - soup = BeautifulSoup(html.data, 'html.parser') - - # Get the "vung-doc" class, this contains a url for each page, - # which redirects to the source of the image. - # Deletes the first and last items, since they're trash. - soup_img = soup.find_all('img') - del soup_img[0] - del soup_img[len(soup_img)-1] - - # Stores each image url in a list. - image_url_list = [] - for img in soup_img: - # Gets the sring of the url, splits it by the char '/', - # and gets the last item, which is the name of the file. - - image_url_list.append([img['src'], img['src'].split('/')[-1]]) - - return image_url_list diff --git a/tirante/gcl.py b/tirante/gcl.py deleted file mode 100644 index 6956e7c..0000000 --- a/tirante/gcl.py +++ /dev/null @@ -1,87 +0,0 @@ -"""MIT License - -Copyright (c) 2019 David Luevano - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" -import urllib3 -from bs4 import BeautifulSoup - - -def get_chapters_list(main_url, - manga_name_url, - manga_name, - reverse_sorted=True): - """ - Retrieves chapter urls and names. Returns a list of lists - containing the url and the title of the chapter. - main_url: Main webpage name (source). - manga_name_url: Name of the manga in the url format - that's used by the webpage. - manga_name: Actual name of the manga, as it appears in the webpage. - reverse_sorted: Sorting of the final array. - """ - - manga_url = ''.join([main_url, manga_name_url]) - - # Not actually a file, but the content of the html. - html = urllib3.PoolManager().request('GET', manga_url) - - # Get the data from the html and parse it. - soup = BeautifulSoup(html.data, 'html.parser') - - # Get the "rows" class, this contains the url - # and title data for each chapter. - # Deletes the first tag, since it's not useful. - soup_rows = soup.find_all('div', {'class': 'row'}) - del soup_rows[0] - - # Creates a list to store date for each url and chapter name. - chapter_list = [] - - for row in soup_rows: - - # Gets the url name from the a tag. - href = row.a['href'] - # Same, for the title. Deletes every ocurrance of the manga name, - # unwanted characters and then gets everyword. - title_words = row.a['title'].replace(manga_name, '').replace('?', '') - title_words = title_words.replace(':', '').replace('-', '') - title_words = title_words.replace('...', '').replace(',', '').split() - - # Doing all the work in oneliner doesn't work for some chapters, - # for some reason. - # title = '_'.join(row.a['title'].replace(manga_name, '') - # .replace(':', '').replace('-', '').lower().split()) - - # Lowers every word and appends it to a new list, - # then it gets joined with '_' as a sep. - title_words_lower = [] - for word in title_words: - title_words_lower.append(word.lower()) - - title = '_'.join(title_words_lower) - - # print(href, title) - chapter_list.append([href, title]) - - if reverse_sorted: - return chapter_list[::-1] - else: - return chapter_list diff --git a/tirante/get_chapter_image_list.py b/tirante/get_chapter_image_list.py new file mode 100644 index 0000000..dbe4572 --- /dev/null +++ b/tirante/get_chapter_image_list.py @@ -0,0 +1,57 @@ +"""MIT License + +Copyright (c) 2019 David Luevano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" +import urllib3 +from bs4 import BeautifulSoup + + +def get_chapter_image_list(chapter_data): + """ + Gets the links for each image in the chapter, + and returns a list of the links. + Returns a list of the image urls and its file name. + chapter_data: A list containing a url and a title. + NOTE: Not for direct use with the result of 'get_chapters_list' + """ + + # Not actually a file, but the content of the html. + html = urllib3.PoolManager().request('GET', chapter_data[0]) + + # Get the data from the html and parse it. + soup = BeautifulSoup(html.data, 'html.parser') + + # Get the "vung-doc" class, this contains a url for each page, + # which redirects to the source of the image. + # Deletes the first and last items, since they're trash. + soup_img = soup.find_all('img') + del soup_img[0] + del soup_img[len(soup_img)-1] + + # Stores each image url in a list. + image_url_list = [] + for img in soup_img: + # Gets the sring of the url, splits it by the char '/', + # and gets the last item, which is the name of the file. + + image_url_list.append([img['src'], img['src'].split('/')[-1]]) + + return image_url_list diff --git a/tirante/get_chapters_list.py b/tirante/get_chapters_list.py new file mode 100644 index 0000000..6956e7c --- /dev/null +++ b/tirante/get_chapters_list.py @@ -0,0 +1,87 @@ +"""MIT License + +Copyright (c) 2019 David Luevano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" +import urllib3 +from bs4 import BeautifulSoup + + +def get_chapters_list(main_url, + manga_name_url, + manga_name, + reverse_sorted=True): + """ + Retrieves chapter urls and names. Returns a list of lists + containing the url and the title of the chapter. + main_url: Main webpage name (source). + manga_name_url: Name of the manga in the url format + that's used by the webpage. + manga_name: Actual name of the manga, as it appears in the webpage. + reverse_sorted: Sorting of the final array. + """ + + manga_url = ''.join([main_url, manga_name_url]) + + # Not actually a file, but the content of the html. + html = urllib3.PoolManager().request('GET', manga_url) + + # Get the data from the html and parse it. + soup = BeautifulSoup(html.data, 'html.parser') + + # Get the "rows" class, this contains the url + # and title data for each chapter. + # Deletes the first tag, since it's not useful. + soup_rows = soup.find_all('div', {'class': 'row'}) + del soup_rows[0] + + # Creates a list to store date for each url and chapter name. + chapter_list = [] + + for row in soup_rows: + + # Gets the url name from the a tag. + href = row.a['href'] + # Same, for the title. Deletes every ocurrance of the manga name, + # unwanted characters and then gets everyword. + title_words = row.a['title'].replace(manga_name, '').replace('?', '') + title_words = title_words.replace(':', '').replace('-', '') + title_words = title_words.replace('...', '').replace(',', '').split() + + # Doing all the work in oneliner doesn't work for some chapters, + # for some reason. + # title = '_'.join(row.a['title'].replace(manga_name, '') + # .replace(':', '').replace('-', '').lower().split()) + + # Lowers every word and appends it to a new list, + # then it gets joined with '_' as a sep. + title_words_lower = [] + for word in title_words: + title_words_lower.append(word.lower()) + + title = '_'.join(title_words_lower) + + # print(href, title) + chapter_list.append([href, title]) + + if reverse_sorted: + return chapter_list[::-1] + else: + return chapter_list diff --git a/tirante/up_dat.py b/tirante/up_dat.py deleted file mode 100644 index 5834563..0000000 --- a/tirante/up_dat.py +++ /dev/null @@ -1,101 +0,0 @@ -"""MIT License - -Copyright (c) 2019 David Luevano - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import os - -# Project specific imports. -from gcl import get_chapters_list -from ch_man import chapters_csv_to_list -from chim_man import chapter_image_list_to_csv - - -def update_database(main_url, - manga_name_url, - manga_name, - manga_dir, - manga_data_dir): - """ - Updates the database already created, adding missing ones. - main_url: Main webpage name (source). - manga_name_url: Name of the manga in the url format - that's used by the webpage. - manga_name: Actual name of the manga, as it appears in the webpage. - manga_dir: Main manga folder in computer, subfolders here will be created. - manga_data_dir: Main manga data folder in computer. - """ - - # A better "naming" for the manga, for use with folder creation. - # As well as the name of the main database. - m_name = '_'.join(word.lower() for word in manga_name.split()) - m_name_ext = ''.join([m_name, '.csv']) - - # Navigate to where the main data folder is, - # then to where the manga folder is. - os.chdir(manga_data_dir) - try: - os.mkdir(m_name) - os.chdir(m_name) - except FileExistsError: - print(''.join([m_name, - ' folder already exists.'])) - os.chdir(m_name) - - # Get a list of files present in path. - data_list_dir = os.listdir() - - # First, download the data from the web. - new_chapter_list = get_chapters_list(main_url=main_url, - manga_name_url=manga_name_url, - manga_name=manga_name, - reverse_sorted=False) - - # And then, read the current database. - last_chapter = chapters_csv_to_list(m_name_ext)[-1] - - # The missing chapters. - missing_chapters = [] - for chapter in new_chapter_list: - # If we get to the last acquired chapter, exit loop. - if chapter == last_chapter: - break - missing_chapters.append(chapter) - - # Reverse the order. - missing_chapters = missing_chapters[::-1] - - # Write the last chapters to already existing csv file. - # No need for checking if items are present, - # since that's checked on the last steps, - # that's how missing_chapters is acquired. - with open(m_name_ext, 'a') as outcsv: - for chapter in missing_chapters: - outcsv.write(''.join([chapter[0], ',', chapter[1], '\n'])) - - # Create the missing csv data files for each chapter. - for chapter in missing_chapters: - # Get the list for the images of each chapter. - chapter_name_ext = ''.join([chapter[1], '.csv']) - if chapter_name_ext not in data_list_dir: - chapter_image_list_to_csv(chapter) - else: - print(''.join([chapter_name_ext, ' already exists.'])) diff --git a/tirante/update_database.py b/tirante/update_database.py new file mode 100644 index 0000000..0950cda --- /dev/null +++ b/tirante/update_database.py @@ -0,0 +1,101 @@ +"""MIT License + +Copyright (c) 2019 David Luevano + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import os + +# Project specific imports. +from get_chapters_list import get_chapters_list +from chapters_manager import chapters_csv_to_list +from chapter_images_manager import chapter_images_list_to_csv + + +def update_database(main_url, + manga_name_url, + manga_name, + manga_dir, + manga_data_dir): + """ + Updates the database already created, adding missing ones. + main_url: Main webpage name (source). + manga_name_url: Name of the manga in the url format + that's used by the webpage. + manga_name: Actual name of the manga, as it appears in the webpage. + manga_dir: Main manga folder in computer, subfolders here will be created. + manga_data_dir: Main manga data folder in computer. + """ + + # A better "naming" for the manga, for use with folder creation. + # As well as the name of the main database. + m_name = '_'.join(word.lower() for word in manga_name.split()) + m_name_ext = ''.join([m_name, '.csv']) + + # Navigate to where the main data folder is, + # then to where the manga folder is. + os.chdir(manga_data_dir) + try: + os.mkdir(m_name) + os.chdir(m_name) + except FileExistsError: + print(''.join([m_name, + ' folder already exists.'])) + os.chdir(m_name) + + # Get a list of files present in path. + data_list_dir = os.listdir() + + # First, download the data from the web. + new_chapter_list = get_chapters_list(main_url=main_url, + manga_name_url=manga_name_url, + manga_name=manga_name, + reverse_sorted=False) + + # And then, read the current database. + last_chapter = chapters_csv_to_list(m_name_ext)[-1] + + # The missing chapters. + missing_chapters = [] + for chapter in new_chapter_list: + # If we get to the last acquired chapter, exit loop. + if chapter == last_chapter: + break + missing_chapters.append(chapter) + + # Reverse the order. + missing_chapters = missing_chapters[::-1] + + # Write the last chapters to already existing csv file. + # No need for checking if items are present, + # since that's checked on the last steps, + # that's how missing_chapters is acquired. + with open(m_name_ext, 'a') as outcsv: + for chapter in missing_chapters: + outcsv.write(''.join([chapter[0], ',', chapter[1], '\n'])) + + # Create the missing csv data files for each chapter. + for chapter in missing_chapters: + # Get the list for the images of each chapter. + chapter_name_ext = ''.join([chapter[1], '.csv']) + if chapter_name_ext not in data_list_dir: + chapter_images_list_to_csv(chapter) + else: + print(''.join([chapter_name_ext, ' already exists.'])) -- cgit v1.2.3-54-g00ecf