Disable stream buffering on upstream nginx server

with chrome profiles

Dockerfile

@@ -1,21 +1,42 @@
 FROM python:3.10
 ENV DEBIAN_FRONTEND noninteractive
 WORKDIR /app/
-RUN apt update \
-    && wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb \
-    && dpkg -i libssl1.1_1.1.0g-2ubuntu4_amd64.deb \
-    && apt install redis libgtk-4-1 libvulkan1 libxdamage1 -y \
-    && curl -sSLO https://nc.piair.xyz/s/BKLsBWoZkTdYjfq/download/chrome.deb \
-    && ln -fs /usr/share/zoneinfo/Europe/Paris /etc/localtime \
-    && git clone https://gitea.augustin64.fr/piair/MsRewards-Reborn \
-    && python3 -m pip install -r MsRewards-Reborn/requirements.txt \
-    && wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key \
-    && curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
-    && echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" | tee -a /etc/apt/sources.list.d/grafana.list \
-    && apt update \
-    && apt install novnc websockify grafana xvfb nginx nano tzdata sqlite3 apt-transport-https software-properties-common wget wfrench tigervnc-standalone-server libasound2 libatk-bridge2.0-0 libnss3 libnspr4 xvfb libgbm1 libatk1.0-0 libu2f-udev libatspi2.0-0 libcups2 libxkbcommon0 libxrandr2 libdbus-1-3 xdg-utils fonts-liberation libdrm2 -y \
-    && bash MsRewards-Reborn/config/config.sh \
+
+# Initial apt install
+RUN apt update
+RUN apt install -y libgtk-4-1 libvulkan1 libxdamage1 \
+                novnc websockify xvfb nginx nano tzdata \
+                sqlite3 apt-transport-https software-properties-common \
+                wget wfrench tigervnc-standalone-server libasound2 \
+                libatk-bridge2.0-0 libnss3 libnspr4 xvfb libgbm1 libatk1.0-0 \
+                libu2f-udev libatspi2.0-0 libcups2 libxkbcommon0 libxrandr2 \
+                libdbus-1-3 xdg-utils fonts-liberation libdrm2
+
+# Additional repos and packages
+RUN wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4_amd64.deb \
+    && dpkg -i libssl1.1_1.1.0g-2ubuntu4_amd64.deb
+RUN curl -sSL http://mirror.cs.uchicago.edu/google-chrome/pool/main/g/google-chrome-stable/google-chrome-stable_123.0.6312.86-1_amd64.deb -o chrome.deb \
     && dpkg -i chrome.deb
+RUN ln -fs /usr/share/zoneinfo/Europe/Paris /etc/localtime
+RUN wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key \
+    && echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" | tee -a /etc/apt/sources.list.d/grafana.list
+RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+# Install from new repo
+RUN apt update \
+    && apt install -y redis grafana
+
+# Configure Grafana
+RUN grafana-cli plugins install frser-sqlite-datasource
+
+COPY requirements.txt /app/requirements.txt
+RUN python3 -m pip install -r requirements.txt
+
+# Setup app
+RUN git clone https://gitea.augustin64.fr/piair/MsRewards-Reborn
+# Use this instead when developping locally:
+# COPY . /app/MsRewards-Reborn
+
+RUN bash MsRewards-Reborn/config/config.sh
 
 ENV TZ="Europe/Paris"
 WORKDIR /app/MsRewards-Reborn/Flask/

Flask/app.py

@@ -373,8 +373,18 @@ def config_post():
 def logs():
     with open("/app/MsRewards-Reborn/user_data/configs.json", "r") as inFile:
         configs = json.load(inFile)
-    print(configs)
-    return(render_template("logs.html", data=configs))
+    
+    files = [(configs[i]["name"], i) for i in configs]
+    config_files = [i[1] for i in files]
+    for f in os.listdir("/app/MsRewards-Reborn/Flask/static/logs"):
+        fid = ".".join(f.split(".")[:-1]) # filename without .txt
+        if f != ".gitignore" and fid not in config_files:
+            files.append((f, fid))
+
+    return render_template(
+        "logs.html",
+        files=files
+    )
 
 
 @app.route("/stats/", methods=["GET", "POST"])

Flask/templates/logs.html

@@ -8,14 +8,37 @@
 
 <select name="select" onchange="change_logs(this.value)">
     <option id="null" value="0">Choisir une config</option>
-    {% for i in data %}
-    <option id="{{data[i]['name']}}" value="{{i}}">{{data[i]['name']}}</option>
+    {% for file in files %}
+    <option id="{{ file[0] }}" value="{{ file[1] }}">{{ file[0] }}</option>
     {% endfor %}
 </select>
 <br><br>
-<iframe type="text/html" src="{{url_for('static', filename='logs/1.txt')}}"  width="100%" height="85%" id="embed">
+<iframe type="text/html" src="{{url_for('static', filename='logs/1.txt')}}"  width="100%" height="85%" id="embed"></iframe>
 
+<script defer>
+    const iframe = document.getElementsByTagName("iframe")[0];
+    var script = document.createElement('script');
 
+    // Wait until ansi_up load
+    script.onload = function () {
+        // Wait until iframe load
+        iframe.onload = function() {
+            const subdoc = iframe.contentWindow.document;
+            const subBody = subdoc.getElementsByTagName("body")[0]
+            let ansiOutput = subBody;
+            // Depending on the content encoding (and maybe on the browser)
+            // a <pre> is added around the content of the file
+            if (subBody.getElementsByTagName("pre").length > 0) {
+                ansiOutput = subBody.getElementsByTagName("pre")[0];
+            }
+            const ansi_up = new AnsiUp();
+            ansiOutput.innerHTML = ansi_up.ansi_to_html(ansiOutput.innerText);
+        }
+    };
+    
+    script.src = "https://cdn.jsdelivr.net/npm/ansi_up@4.0.4/ansi_up.js";
+    document.head.appendChild(script);
+</script>
 
 {% endif %}
 {% endblock %}

V6.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3.10
+#!/usr/bin/python3
 from modules.Classes.Config import Config
 from modules.Classes.DiscordLogger import DiscordLogger
 from modules.Classes.UserCredentials import UserCredentials
@@ -9,6 +9,7 @@ from modules.db import add_to_database
 from modules.driver_tools import *
 from modules.error import *
 
+from selenium.common.exceptions import NoSuchElementException
 
 # create a webdriver 
 def create_driver(mobile=False):
@@ -22,16 +23,33 @@ def create_driver(mobile=False):
         "AppleWebKit/537.36 (KHTML, like Gecko)"
         "Chrome/22 Mobile Safari/537.36"
     )
-    chrome_options = webdriver.ChromeOptions()
+
+    chrome_profile_dir = init_profile(config.UserCredentials.get_mail())
+
+    # Full list on https://github.com/GoogleChrome/chrome-launcher/blob/main/docs/chrome-flags-for-tools.md
+    arguments = [
+        "--no-first-run",
+        "--ash-no-nudges",
+        "--no-default-browser-check",
+        "--disable-features=PrivacySandboxSettings4,Translate",
+        "--disable-search-engine-choice-screen",
+        f"--user-data-dir={chrome_profile_dir}/"
+    ]
+
     if mobile:
-        chrome_options.add_argument(f"--user-agent={mobile_user_agent}")
+        arguments.append(f"--user-agent={mobile_user_agent}")
     else:
-        chrome_options.add_argument(f"--user-agent={pc_user_agent}")
+        arguments.append(f"--user-agent={pc_user_agent}")
+
     # disabled as it may cause detection
     if config.proxy.is_enabled():
-        chrome_options.add_argument(f'--proxy-server={config.proxy.ip}:{config.proxy.port}')
+        arguments.append(f'--proxy-server={config.proxy.ip}:{config.proxy.port}')
+
+    chrome_options = webdriver.ChromeOptions()
+    for arg in arguments:
+        chrome_options.add_argument(arg)
+
     driver = uc.Chrome(options=chrome_options)
-    set_language(driver)
     return driver
 
 
@@ -152,7 +170,7 @@ def play_quiz4(override: int = None):
     except Exception as e:
         log_error(e)
         raise ValueError(e)
-    info("Quiz 8 done.")
+    info("Quiz 4 done.")
     custom_sleep(3)
 
 
@@ -165,6 +183,8 @@ def do_poll():
         try:
             answer_elem.click()
         except exceptions.ElementNotInteractableException:
+            warning("element not clickable. Waiting a bit and retrying.")
+            custom_sleep(uniform(2, 2.5))
             driver.execute_script("arguments[0].click();", answer_elem)
         custom_sleep(uniform(2, 2.5))
     except Exception as err:
@@ -190,7 +210,7 @@ def all_cards():
         info("no promo card")
 
     if len(card_list) < 10:  # most likely an error during loading
-        if "suspendu" in driver.page_source:
+        if "suspendu" in driver.page_source or "suspended" in driver.page_source:
             raise Banned()
         driver.refresh()
         card_list = driver.find_elements(By.CLASS_NAME, "c-card-content")
@@ -360,20 +380,33 @@ def login_part_1():
     driver = config.WebDriver.driver
     driver.get("https://login.live.com")
     wait_until_visible(By.ID, "i0116", browser=driver)
-    mail_elem = driver.find_element(By.ID, "i0116")
-    send_keys_wait(mail_elem, config.UserCredentials.get_mail())
-    mail_elem.send_keys(Keys.ENTER)
+    send_wait_and_confirm(
+        driver.find_element(By.ID, "i0116"),
+        config.UserCredentials.get_mail()
+    )
     wait_until_visible(By.ID, "i0118", browser=driver)
-    pwd_elem = driver.find_element(By.ID, "i0118")
-    send_keys_wait(pwd_elem, config.UserCredentials.get_password())
-    pwd_elem.send_keys(Keys.ENTER)
-    custom_sleep(2)
+    send_wait_and_confirm(
+        driver.find_element(By.ID, "i0118"),
+        config.UserCredentials.get_password()
+    )
     # 2FA
-    if "Entrez le code de sécurité" in driver.page_source:
     try:
-            a2f_elem = driver.find_element(By.ID, "idTxtBx_SAOTCC_OTC")
-            a2f_elem.send_keys(config.UserCredentials.get_tfa().now())
-            a2f_elem.send_keys(Keys.ENTER)
+        if not wait_until_visible(By.ID, "idTxtBx_SAOTCC_OTC", browser=driver, timeout=5, raise_error=False):
+            custom_sleep(2)
+            return
+
+        tfa = config.UserCredentials.get_tfa()
+        if tfa is None:
+            error("2FA needed but no code available for this account, sending error")
+            raise ValueError("2FA needed but no code available for this account")
+        else:
+            a2f_code = tfa.now()
+
+        info(f"Need 2FA, I have code: {a2f_code}")
+        send_wait_and_confirm(
+            driver.find_element(By.ID, "idTxtBx_SAOTCC_OTC"),
+            a2f_code
+        )
     except Exception as err:
         log_error(err)
 
@@ -410,8 +443,16 @@ def login_part_2():
 # login() tries to login to your Microsoft account.
 # it uses global variable g._mail and g._password to login
 def login():
+    def logged_in():
+        driver.get("https://login.live.com")
+        custom_sleep(10)
+        if get_domain(driver) == "account.microsoft.com":
+            return True
+        return False
+
     driver = config.WebDriver.driver
     try:
+        if not logged_in():
             login_part_1()
         login_part_2()
         driver.get("https://rewards.bing.com/")
@@ -610,9 +651,11 @@ def daily_routine(cred: UserCredentials, custom=False):
     try:
         if not custom:  # custom already is logged in
             login()
+
     except Banned:
         log_error("This account is locked.")
-        return
+        raise Banned()
+
     except Identity:
         log_error("This account has an issue.")
         return
@@ -666,7 +709,7 @@ def json_start(json_entry, cred: UserCredentials):
             config.WebDriver.switch_to_driver("PC")
             driver = config.WebDriver.driver
             try:
-                if str(account_id) in json_entry["unban"]:
+                if "unban" in json_entry and str(account_id) in json_entry["unban"]:
                     login_part_1()
                     info("\nGO TO example.com TO PROCEED or wait 1200 secs.")
                     for _ in range(1200):
@@ -680,7 +723,7 @@ def json_start(json_entry, cred: UserCredentials):
                 login()
             try:
                 if str(account_id) in json_entry["tout"]:
-                    daily_routine(cred)
+                    daily_routine(cred, True)
             except KeyError:
                 pass
             else:
@@ -744,12 +787,16 @@ def default_start():
             config.WebDriver.pc_driver.quit()
             config.display.stop()
             break
+        except Banned:
+            warning("this account is banned. Switching to next account")
         except Exception as e:
             log_error(f"Error not caught. Skipping this account. " + format_error(e))
             critical(f"Error not caught. Skipping this account. {e}")
             config.WebDriver.pc_driver.quit()
 
+        finally:
             config.UserCredentials.next_account()
+
     config.display.stop()
 
 

build.sh

@@ -1 +1,12 @@
-sudo docker build --no-cache --network host -t msrewards . && sudo docker run -d --restart unless-stopped -p 1234:1234 -p 2345:2345 -ti --shm-size=2gb --name MsRewards msrewards 
+#!/bin/bash
+
+docker-do () { # Check if sudo needs to be used
+    if id -nG "$(whoami)" | grep -qw "docker"; then
+        docker $@
+    else
+        sudo docker $@
+    fi
+}
+
+docker-do build --network host -t msrewards .
+docker-do run -d --restart unless-stopped -p 1234:1234 -p 2345:2345 -ti --shm-size=2gb --name MsRewards msrewards 

clean.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+docker-do () { # Check if sudo needs to be used
+    if id -nG "$(whoami)" | grep -qw "docker"; then
+        docker $@
+    else
+        sudo docker $@
+    fi
+}
+
+docker-do stop MsRewards
+docker-do rm MsRewards
+docker-do image rm msrewards

config/config.sh

@@ -46,6 +46,7 @@ server {
         proxy_pass         "http://127.0.0.1:6666";
         chunked_transfer_encoding off;
         proxy_buffering off;
+        set_header X-Accel-Buffering no;
     }
     
 }
@@ -59,7 +60,6 @@ sqlite3 /app/MsRewards-Reborn/MsRewards.db "CREATE TABLE comptes (id INTEGER PRI
 printf  "\nconfigurating grafana\n"
 
 cp /app/MsRewards-Reborn/config/grafana.ini /etc/grafana/
-grafana-cli plugins install frser-sqlite-datasource
 
 printf  "setting up default dashboard"
 cp /app/MsRewards-Reborn/config/Stats-dashbord.json /usr/share/grafana/public/dashboards/home.json 

modules/Classes/Config.py

@@ -50,9 +50,14 @@ class Config:
         """
         self.discord = DiscordConfig()
         self.discord.avatar_url = settings["avatarlink"]
-        self.discord.wh_link = discord[config[args.config]["discord"]]["errorsL"]
 
-        if self.discord.wh_link != "":
+        if (
+            "discord" in config[args.config]
+            and config[args.config]["discord"] in discord
+            and "errorsL" in discord[config[args.config]["discord"]]
+            and discord[config[args.config]["discord"]]["errorsL"] != ""
+            ):
+            self.discord.wh_link = discord[config[args.config]["discord"]]["errorsL"]
             self.discord.wh = Webhook.from_url(self.discord.wh_link, adapter=RequestsWebhookAdapter())
         else:
             self.discord.wh = FakeWebHook()

modules/Classes/DiscordConfig.py

@@ -9,6 +9,6 @@ class DiscordConfig:
 
 
 class FakeWebHook:
-    def send(self, *args):
-        debug(f"Used a webhook call without webhook url with {args}")
+    def send(self, *args, **kwargs):
+        debug(f"Used a webhook call without webhook url with {args} {kwargs}")
         

modules/Classes/DiscordLogger.py

@@ -30,7 +30,7 @@ class DiscordLogger:
         )
         file = File("screenshot.png")
         embed.set_image(url="attachment://screenshot.png")
-        embed.set_footer(text=self.config.UserCredentials.creds.get_mail())
+        embed.set_footer(text=self.config.UserCredentials.get_mail() + " - " + self.config.WebDriver.current_driver())
 
         self.config.discord.wh.send(embed=embed, username="error", file=file, avatar_url=self.config.discord.avatar_url)
         self.config.discord.wh.send(username="error", file=File("page.html"), avatar_url=self.config.discord.avatar_url)

modules/Classes/Driver.py

@@ -20,3 +20,6 @@ class Driver:
 
             case _:
                 raise ValueError("The driver must be either pc or mobile")
+
+    def current_driver(self):
+        return "PC" if self.pc_driver == self.driver else "Mobile"

modules/Classes/UserCredentials.py

@@ -30,8 +30,9 @@ class UserCredentials:
 
     def get_tfa(self):
         if not self.tfa_enable():
-            warning("Warning: TFA is not enabled. Calling get_tfa is an expected behaviour.")
-        return TOTP(self.data[self.current]["tfa"])
+            warning("Warning: TFA is not enabled. Can't get a TFA code.")
+            return None
+        return TOTP(self.data[self.current]["2fa"])
 
     def next_account(self):
         self.current += 1
@@ -41,4 +42,5 @@ class UserCredentials:
             debug("No new credentials.")
 
     def is_valid(self):
-        return self.current < self.total
+        return (self.current < self.total
+                and self.get_mail() != "" and self.get_mail is not None)

modules/Tools/generate_error.py

@@ -0,0 +1,8 @@
+import undetected_chromedriver as uc
+from pyvirtualdisplay.smartdisplay import SmartDisplay
+
+display = SmartDisplay(size=(1920, 1080))
+display.start()
+driver = uc.Chrome()
+driver.close()
+driver.close()

modules/Tools/logger.py

@@ -30,7 +30,7 @@ class ColoredFormatter(logging.Formatter):
 
 # Set up the root logger
 root_logger = logging.getLogger(__name__)
-root_logger.setLevel(logging.DEBUG)
+root_logger.setLevel(logging.INFO)
 
 # Create a console handler and set the formatter
 ch = logging.StreamHandler()

modules/Tools/tools.py

@@ -10,7 +10,7 @@ def get_domain(driver):
 
 def custom_sleep(temps):
     try:
-        if True:  # todo: change this awful condition
+        if False:  # todo: change this awful condition
             points = ["⢿", "⣻", "⣽", "⣾", "⣷", "⣯", "⣟", "⡿"]
             passe = 0
             for _ in range(int(temps)):

modules/Tools/update_chrome.py

@@ -0,0 +1,48 @@
+import requests
+import re
+from packaging import version
+import subprocess
+
+from logger import critical, info, error
+
+errorMessage = subprocess.run(['python3', 'generate_error.py'], check=False, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE).stderr.decode("utf-8")
+
+versionPattern = "This version of ChromeDriver only supports Chrome version ([0-9]+)"
+
+try:
+    versionN = re.search(versionPattern, errorMessage)[1]
+except Exception as e:
+    critical("Can't get version number from error")
+    error(e)
+    exit(0)
+
+info(f"Needed version : '{versionN}'")
+
+downloadUrl = "http://mirror.cs.uchicago.edu/google-chrome/pool/main/g/google-chrome-stable/"
+r = requests.get(downloadUrl)
+
+content = r.text
+
+exactVersionList = re.findall(f"(google-chrome-stable_({versionN}.[0-9.]+)[^<^>^\"]+)", content)
+
+try:
+    best = exactVersionList[0]
+except Exception as e:
+    critical("No version matches required version")
+    error(e)
+    exit(0)
+
+for i in exactVersionList:
+    if version.parse(i[1]) > version.parse(best[1]):
+        best = i
+
+chromeDebURL = f"http://mirror.cs.uchicago.edu/google-chrome/pool/main/g/google-chrome-stable/{best[0]}"
+info(f"chrome deb URL : {chromeDebURL}")
+info("downloading chrome")
+
+subprocess.call(['wget', "-O", "/tmp/chrome.deb", chromeDebURL])
+info("Chrome deb downloaded. Installing chrome")
+
+subprocess.call(["dpkg", "-i", "/tmp/chrome.deb"])
+info("Chrome installed")

modules/driver_tools.py

@@ -1,3 +1,5 @@
+import os
+import json
 from random import uniform
 
 from selenium.common import TimeoutException
@@ -10,33 +12,36 @@ from modules.Tools.logger import debug
 from modules.Tools.tools import *
 
 
-def set_language(ldriver):
-    ldriver.get("chrome://settings/languages")
-    action = ActionChains(ldriver)
-    action.reset_actions()
-    # select language
-    x_coord = 1200
-    y_coord = 150
-    action.move_by_offset(x_coord, y_coord).click().perform()
-    sleep(0.5)
-    # scroll down
-    action.reset_actions()
-    elm = ldriver.find_element(By.XPATH, "/html/body")
-    ActionChains(ldriver) \
-        .send_keys("french") \
-        .pause(0.5) \
-        .send_keys(Keys.TAB + Keys.TAB + Keys.ENTER + Keys.TAB + Keys.TAB + Keys.ENTER) \
-        .perform()
-    x_coord = 1163
-    y_coord = 717
-    action.move_by_offset(x_coord, y_coord).click().perform()
-    # put to the top
-    sleep(0.5)
-    action.reset_actions()
-    x_coord = 1257
-    y_coord = 328
-    action.move_by_offset(x_coord, y_coord).click().perform()
-    action.click().perform()
+def init_profile(mail):
+    chrome_profile_dir = "/app/MsRewards-Reborn/user_data/profile/"+mail
+    os.makedirs(chrome_profile_dir, exist_ok=True)
+
+    preferences_file = os.path.join(chrome_profile_dir, "Default", "Preferences")
+    if not os.path.exists(preferences_file):
+        os.makedirs(os.path.join(chrome_profile_dir, "Default"), exist_ok=True)
+        with open(preferences_file, "w") as f:
+            json.dump(
+                {
+                    "intl": {
+                        "accept_languages": "fr-FR,en-US,en",
+                        "selected_languages": "fr-FR,en-US,en"
+                    }
+               }, f
+            )
+    else:
+        with open(preferences_file, "r") as f:
+            settings = json.load(f)
+        
+        if "intl" not in settings:
+            settings["intl"] = {}
+
+        settings["intl"]["accept_languages"] = "fr-FR,en-US,en"
+        settings["intl"]["selected_languages"] = "fr-FR,en-US,en"
+
+        with open(preferences_file, "w") as f:
+            json.dump(settings, f)
+
+    return chrome_profile_dir
 
 
 # Deal with RGPD popup as well as some random popup like 'are you satisfied' one
@@ -60,13 +65,18 @@ def send_keys_wait(element, keys: str) -> None:
         element.send_keys(i)
         sleep(uniform(0.1, 0.3))
 
+def send_wait_and_confirm(element, keys: str) -> None:
+    send_keys_wait(element, keys)
+    element.send_keys(Keys.ENTER)
+
 
 # Wait for the presence of the element identifier or [timeout]s
-def wait_until_visible(search_by: str, identifier: str, timeout: int = 20, browser=None) -> bool:
+def wait_until_visible(search_by: str, identifier: str, timeout: int = 20, browser=None, raise_error=True) -> bool:
     try:
         WebDriverWait(browser, timeout).until(
             expected_conditions.visibility_of_element_located((search_by, identifier)), "element not found")
         return True
     except TimeoutException as e:
+        if raise_error:
             error(f"element {identifier} not found after {timeout}s")
         return False

version

@@ -1 +1 @@
-v6.8.38
+v6.8.45