Added scripts to scrape PDF's, make table operations, convert coordinates between different formats
This commit is contained in:
TheFGFSEagle
63
misc/scrape-emanualonline-pdf.py
Normal file
63
misc/scrape-emanualonline-pdf.py
Normal file
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python
|
||||
#-*- coding:utf-8 -*-
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import re
|
||||
import requests
|
||||
|
||||
from fgtools.utils import constants
|
||||
|
||||
pattern = r'(?<=src=")https:\/\/static-repo.emanualonline.com\/.+\.jpg(?=")'
|
||||
|
||||
def download_pages(url, output):
|
||||
html = requests.get(url).text
|
||||
urls = re.findall(pattern, html)
|
||||
urltemplate = "/".join(urls[0].split("/")[:-2] + ["%d", "%d.jpg"])
|
||||
|
||||
paths = []
|
||||
i = 1
|
||||
while True:
|
||||
page = requests.get(urltemplate % (i, i))
|
||||
i += 1
|
||||
if page.status_code != 200:
|
||||
break
|
||||
|
||||
path = os.path.join(constants.CACHEDIR, os.path.split(output)[-1] + f"-{i}.jpg")
|
||||
paths.append(path)
|
||||
with open(path, "wb") as f:
|
||||
f.write(page.content)
|
||||
|
||||
return paths
|
||||
|
||||
def write_pdf(paths, output):
|
||||
print(f"Joining {len(paths)} JPG files into {output} … ", end="")
|
||||
newpaths = " ".join([f'"{path}"' for path in paths])
|
||||
os.system(f'img2pdf {newpaths} --output "{output}"')
|
||||
print("done.")
|
||||
print("Deleting JPG files … ", end="")
|
||||
for path in paths:
|
||||
os.remove(path)
|
||||
print("done")
|
||||
|
||||
if __name__ == "__main__":
|
||||
argp = argparse.ArgumentParser()
|
||||
|
||||
argp.add_argument(
|
||||
"url",
|
||||
help="URL to emanualonline.com PDF offer"
|
||||
)
|
||||
|
||||
argp.add_argument(
|
||||
"-o", "--output",
|
||||
help="Output file",
|
||||
required=True
|
||||
)
|
||||
|
||||
args = argp.parse_args()
|
||||
|
||||
os.makedirs(os.path.join(*os.path.split(os.path.relpath(args.output))[:-1]) or ".", exist_ok=True)
|
||||
|
||||
paths = download_pages(args.url, args.output)
|
||||
write_pdf(paths, args.output)
|
||||
|
||||
112
misc/scrape-scribd.py
Normal file
112
misc/scrape-scribd.py
Normal file
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python
|
||||
#-*- coding:utf-8 -*-
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import re
|
||||
import requests
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from fgtools.utils import constants
|
||||
json_pattern = r'(?<=content-url: ")https:\/\/html.scribdassets.com\/.+\.jsonp(?=")'
|
||||
img_pattern = r'<img .+?\/>'
|
||||
|
||||
class JSPage:
|
||||
def __init__(self, number, width, height, url):
|
||||
self.number = number
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.url = url
|
||||
|
||||
def get_image(self):
|
||||
text = requests.get(self.url).text
|
||||
images = list(map(lambda s: BeautifulSoup(s.replace("\\", ""), features="lxml").body.find("img"), re.findall(img_pattern, text)))
|
||||
print(len(images))
|
||||
src_image = Image.open(requests.get(images[0]["orig"], stream=True).raw)
|
||||
pil_image = Image.new("RGB", (self.width, self.height))
|
||||
pil_image.paste((255, 255, 255), (0, 0, pil_image.size[0], pil_image.size[1]))
|
||||
for image in images:
|
||||
style = {}
|
||||
for item in image["style"].split(";"):
|
||||
item = item.split(":")
|
||||
style[item[0]] = item[1].replace("px", "")
|
||||
|
||||
clip = style["clip"]
|
||||
clip = {k: int(v) for k, v in zip(("top", "right", "bottom", "left"), clip[clip.find("(") + 1:-1].split(" "))}
|
||||
cropped_src_image = src_image.copy().crop((clip["left"], clip["top"], clip["right"], clip["bottom"]))
|
||||
pil_image.paste(cropped_src_image, (int(style["left"]) + clip["left"], int(style["top"]) + clip["top"]))
|
||||
|
||||
return pil_image
|
||||
|
||||
def parse_pages_script(script):
|
||||
lines = list(map(str.strip, script.split("\n")[1:-1]))
|
||||
number = 0
|
||||
width = 0
|
||||
height = 0
|
||||
url = ""
|
||||
pages = []
|
||||
for line in lines:
|
||||
if "pageNum" in line:
|
||||
number = int(line.split(": ")[1][:-1])
|
||||
elif "origWidth" in line:
|
||||
width = int(line.split(": ")[1][:-1])
|
||||
elif "origHeight" in line:
|
||||
height = int(line.split(": ")[1][:-1])
|
||||
elif "contentUrl" in line:
|
||||
url = line.split(": ")[1][1:-1]
|
||||
|
||||
if number and width and height and url:
|
||||
page = JSPage(number, width, height, url)
|
||||
pages.append(page)
|
||||
number = width = height = 0
|
||||
url = ""
|
||||
|
||||
return pages
|
||||
|
||||
def download_pages(url, output):
|
||||
html = BeautifulSoup(requests.get(url).text, features="lxml")
|
||||
pages_script = html.body.find("div", attrs={"class": "outer_page_container"}).find("script", attrs={"type": "text/javascript"})
|
||||
pages = sorted(parse_pages_script(str(pages_script)), key=lambda p: p.number)
|
||||
|
||||
paths = []
|
||||
for page in pages:
|
||||
path = os.path.join(constants.CACHEDIR, os.path.split(output)[-1] + f"-{page.number}.jpg")
|
||||
paths.append(path)
|
||||
page.get_image().save(path, "JPEG")
|
||||
|
||||
return paths
|
||||
|
||||
def write_pdf(paths, output):
|
||||
print(f"Joining {len(paths)} JPG files into {output} … ", end="")
|
||||
newpaths = " ".join([f'"{path}"' for path in paths])
|
||||
os.system(f'img2pdf {newpaths} --output "{output}"')
|
||||
print("done.")
|
||||
print("Deleting JPG files … ", end="")
|
||||
for path in paths:
|
||||
os.remove(path)
|
||||
print("done")
|
||||
|
||||
if __name__ == "__main__":
|
||||
argp = argparse.ArgumentParser()
|
||||
|
||||
argp.add_argument(
|
||||
"url",
|
||||
help="URL to Scribd web page"
|
||||
)
|
||||
|
||||
argp.add_argument(
|
||||
"-o", "--output",
|
||||
help="Output file",
|
||||
required=True
|
||||
)
|
||||
|
||||
args = argp.parse_args()
|
||||
|
||||
os.makedirs(os.path.join(*os.path.split(os.path.relpath(args.output))[:-1]) or ".", exist_ok=True)
|
||||
|
||||
paths = download_pages(args.url, args.output)
|
||||
write_pdf(paths, args.output)
|
||||
|
||||
146
misc/tabletool.py
Normal file
146
misc/tabletool.py
Normal file
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python
|
||||
#-*- coding:utf-8 -*-
|
||||
|
||||
import argparse
|
||||
|
||||
class TableRow:
|
||||
def __init__(self, cols):
|
||||
self.values = [None] * cols
|
||||
|
||||
def __len__(self):
|
||||
return len(self.values)
|
||||
|
||||
def __setitem__(self, index, value):
|
||||
if index >= len(self.values):
|
||||
raise IndexError(f"column index {index} out of bounds for row with {len(self.values)} columns")
|
||||
self.values[index] = value
|
||||
|
||||
def __getitem__(self, index):
|
||||
if index >= len(self.values):
|
||||
raise IndexError(f"column index {index} out of bounds for row with {len(self.values)} columns")
|
||||
return self.values[index]
|
||||
|
||||
def __imul__(self, other):
|
||||
if len(self.values) != len(other.values):
|
||||
raise IndexError(f"attempting to perform *= on two table rows with different column counts")
|
||||
for i in range(len(self)):
|
||||
self[i] *= other[i]
|
||||
return self
|
||||
|
||||
def __isub__(self, other):
|
||||
if len(self.values) != len(other.values):
|
||||
raise IndexError(f"attempting to perform -= on two table rows with different column counts")
|
||||
for i in range(len(self)):
|
||||
self[i] -= other[i]
|
||||
return self
|
||||
|
||||
def __iadd__(self, other):
|
||||
if len(self.values) != len(other.values):
|
||||
raise IndexError(f"attempting to perform += on two table rows with different column counts")
|
||||
for i in range(len(self)):
|
||||
self[i] += other[i]
|
||||
return self
|
||||
|
||||
class Table:
|
||||
def __init__(self, rows, cols):
|
||||
self.rows = [TableRow(cols) for i in range(rows)]
|
||||
|
||||
def __getitem__(self, index):
|
||||
if index >= len(self.rows):
|
||||
raise IndexError(f"row index {index} out of bounds for table with {len(self.rows)} rows")
|
||||
return self.rows[index]
|
||||
|
||||
def __imul__(self, other):
|
||||
if len(self.rows) != len(other.rows):
|
||||
raise IndexError(f"attempting to perform *= on two tables of different row count")
|
||||
for selfrow, otherrow in zip(self, other):
|
||||
selfrow *= otherrow
|
||||
return self
|
||||
|
||||
def __isub__(self, other):
|
||||
if len(self.rows) != len(other.rows):
|
||||
raise IndexError(f"attempting to perform -= on two tables of different row count")
|
||||
for selfrow, otherrow in zip(self, other):
|
||||
selfrow -= otherrow
|
||||
return self
|
||||
|
||||
def __iadd__(self, other):
|
||||
if len(self.rows) != len(other.rows):
|
||||
raise IndexError(f"attempting to perform += on two tables of different row count")
|
||||
for selfrow, otherrow in zip(self, other):
|
||||
selfrow += otherrow
|
||||
return self
|
||||
|
||||
def add_table(tables):
|
||||
lines = []
|
||||
EOT = False
|
||||
EOF = False
|
||||
while not EOT and not EOF:
|
||||
line = input().strip()
|
||||
if line.lower() == "eot":
|
||||
EOT = True
|
||||
elif line.lower() == "eof":
|
||||
EOF = True
|
||||
else:
|
||||
lines.append(list(filter(None, line.split())))
|
||||
|
||||
if len(lines) > 0:
|
||||
tables.append(lines)
|
||||
|
||||
return EOF
|
||||
|
||||
def parse_tables(tables):
|
||||
newtables = []
|
||||
for table in tables:
|
||||
t = Table(len(table), max(map(len, table)))
|
||||
for i, row in enumerate(table):
|
||||
for j, col in enumerate(table[i]):
|
||||
t[i][j] = float(table[i][j])
|
||||
newtables.append(t)
|
||||
return newtables
|
||||
|
||||
def perform_operation(tables, op):
|
||||
first, *tables = tables
|
||||
for table in tables:
|
||||
if op == "product":
|
||||
first *= table
|
||||
elif op == "difference":
|
||||
first -= table
|
||||
elif op == "sum":
|
||||
first += table
|
||||
return first
|
||||
|
||||
def print_table(table, precision):
|
||||
for row in table:
|
||||
print("\t".join(map(lambda f: str(round(f, precision)), row)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
argp = argparse.ArgumentParser(description="perform various operations on one or more tables")
|
||||
|
||||
argp.add_argument(
|
||||
"-o", "--operation",
|
||||
help="which operation to perform on the inputted tables",
|
||||
required=True,
|
||||
choices=["product", "difference", "sum"]
|
||||
)
|
||||
|
||||
argp.add_argument(
|
||||
"-p", "--precision",
|
||||
help="Number of decimal places the numbers in the outputted table should have",
|
||||
default=6,
|
||||
type=int
|
||||
)
|
||||
|
||||
args = argp.parse_args()
|
||||
|
||||
print("Input as many tables as you want, but at least two - input 'EOF' (without the quotes) when you are done")
|
||||
print("For each table, input as many table rows as you want - input 'EOT' (without the quotes) to end a table")
|
||||
print("For each table row, the columns can be separated by any amount of tabs or spaces")
|
||||
tables = []
|
||||
EOF = False
|
||||
while not EOF:
|
||||
EOF = add_table(tables)
|
||||
tables = parse_tables(tables)
|
||||
table = perform_operation(tables, args.operation)
|
||||
print_table(table, args.precision)
|
||||
|
||||
Reference in New Issue
Block a user