Not getting the Multithreaading to work for GTK 3 python

Debasish Patra · 2021-05-30 22:13:43

Hi,
I am trying to get the themes from the gnome-look.org and trying to create widgets by scraping the website.

I wanted to show the window first while updating the GtkWidgets necessary in the background via another Thread.

Here is my code

#!/usr/bin/python3
# ThemesManager.py
#
# Copyright 2021 Debasish Patra
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, GdkPixbuf
import requests
import sys
import gi
import shutil
from bs4 import BeautifulSoup
import dryscrape
import json
import urllib.parse
import concurrent.futures
import threading

class ReadGnomeLook:
    def format_bytes(self, size):
        # 2**10 = 1024
        power = 2**10
        n = 0
        power_labels = {0 : '', 1: 'KB', 2: 'MB', 3: 'GB'}
        while size > power:
            size /= power
            n += 1
        return str("{:.2f}".format(size)) +' ' + str(power_labels[n])


    def getDownloadLinks(self, childURL):
        #childURL = "https://www.gnome-look.org/s/Gnome/p/1519633"
        childURL = childURL+"#files-panel"
        session = dryscrape.Session()
        session.set_attribute('auto_load_images', False)
        session.visit(childURL)
        response = session.body()
        soup = BeautifulSoup(response, features='lxml')
        downloadlink = []
        allscripts = soup.find_all('script', {"src":False})
        for each_script in range(len(allscripts)):
            content = str(allscripts[each_script]).split("var")
            for indx in content:
                if 'filesJson' in str(indx):
                    content = indx.replace('filesJson = ','').replace(';','')
                    content = json.loads(content)
                    links = []
                    for each_item in content:
                        if each_item['active'] == '1':
                            links.append({'name':each_item['name'],'type':each_item['type'],'size':format_bytes(int(each_item['size'])),'md5sum':each_item['md5sum'],'title':each_item['title'],'description':each_item['description'],'url':urllib.parse.unquote(each_item['url'])})
                    for each in links:
                        downloadlink.append(each)
        return downloadlink

    def readWebpage(self, URL):

        myProducts = []
        baseURL="https://www.gnome-look.org"
        #URL = "https://www.gnome-look.org/browse/cat/132/order/latest/"
        session = dryscrape.Session()
        session.set_header('Host','www.gnome-look.org')
        session.visit(URL)
        response = session.body()
        soup = BeautifulSoup(response, features='lxml')
        #print(soup)
        #soup.find(class="product-browse-item-info")
        mydivs = soup.find_all("div", {"class": "product-browse-item picture"})
        for mydiv in mydivs:
            myProducts.append([
                {'name' : mydiv.div.a.findAll("div",{"class":"product-browse-item-info"})[0].h2.text},
                {'category' : mydiv.div.a.findAll("div",{"class":"product-browse-item-info"})[0].findAll("span")[0].text},
                {'author' : mydiv.div.a.findAll("div",{"class":"product-browse-item-info"})[0].findAll("span")[1].b.text},
                {'img' : mydiv.div.a.div.img['src']},
                {'href' : mydiv.div.a['href']}
            ])
        productCatalog = []
        for elements in  myProducts:
            productCatalog.append([
                {
                    'Name':elements[0]['name'],
                    'Category': elements[1]['category'],
                    'Author': elements[2]['author'],
                    'Image': elements[3]['img'],
                    'Link': baseURL +  elements[4]['href'],
                    #'DownloadLinks': getDownloadLinks(baseURL +  elements[4]['href'])
                }
                ])
        return productCatalog

class AppicationWindow(Gtk.Window):

    def __init__(self):

        Gtk.Window.__init__(self, title="Themes Manager")
       
       # Main Application Window
        # self.set_title("Themes Manager v1.0")
        #self.set_default_size(400, 400)
        self.set_position(Gtk.WindowPosition.CENTER)
        self.connect("destroy",Gtk.main_quit)

        # Create Image and Title Grid
        #self.image = self.getImageFromWeb('https://media.wired.com/photos/592697678d4ebc5ab806acf7/master/w_2560%2Cc_limit/GooglePlay.jpg')
        #self.image.set_from_file("android-download.png")
        #image.set_size(200,200)
        print("Before 1st Show all")
        self.show_all()
        z = threading.Thread(target=self.doProcessing(),daemon=True)
        z.start()
        print("Started Z thread")


    def doProcessing(self):
        # Grid for Full Icon Themes
        self.gridfulliconthemes = Gtk.FlowBox(valign = Gtk.Align.START)
        self.gridfulliconthemesscroll = Gtk.ScrolledWindow(hexpand=True, vexpand=True)    # Create scroll window
        self.gridfulliconthemesscroll.add(self.gridfulliconthemes)       # Adds the TreeView to the scroll container

        self.getProductCatalog()



        ## Start
        self.URLs = []
        self.labels = []
        self.images = []
        self.threads = []

        for each_item in self.productCatalog:
            image = Gtk.Image()
            image.new_from_file('/tmp/82682596e6c89475b2f21221d5dc61927887.png')
            self.images.append(image)
            self.labels.append(Gtk.Label("loading"))

        for each_item in range(0,len(self.productCatalog)):
            #print(each_item[0]['Name'])
            self.URLs.append(self.productCatalog[each_item][0]['Image'])

            vertical_box = Gtk.Box()
            vertical_box.set_homogeneous(True)
            vertical_items = Gtk.FlowBox(valign = Gtk.Align.START)
            vertical_items.set_max_children_per_line(1)

            label = Gtk.Label()
            label.set_text(self.productCatalog[each_item][0]['Name'])
            label.set_line_wrap(True)
            label.set_max_width_chars(10)
            label.set_hexpand(True)
            self.labels.append(label)

            #image = Gtk.Image()
            #self.images.append(image)

            vertical_items.add(self.images[each_item])
            vertical_items.add(self.labels[each_item])

            vertical_box.add(vertical_items)

            vertical_box.connect("button-press-event", self.do_anything)

            self.gridfulliconthemes.add(vertical_box)

        ## End
        

        # Create Notebook to add to the Window
        self.notebook = Gtk.Notebook()
        self.add(self.notebook)

        self.fullicontheme = Gtk.Label()
        self.fullicontheme.set_text("Full Icon Themes")
        self.gtkthemes = Gtk.Label()
        self.gtkthemes.set_text("Gtk 3/4 Themes")
        self.gnomeshellthemes = Gtk.Label()
        self.gnomeshellthemes.set_text("Gnome Shell Themes")

        self.fulliconthemepage = Gtk.Label()
        self.fulliconthemepage.set_text("Full Icon Themes Page")
        self.gtkthemespage = Gtk.Label()
        self.gtkthemespage.set_text("GTK themes Page")
        self.gnomeshellthemespage = Gtk.Label()
        self.gnomeshellthemespage.set_text("Gnome Shell Themes Page")

        #notebook.append_page(fullicontheme, Gtk.Label("Icon Page"))
        self.notebook.append_page(self.gridfulliconthemesscroll, self.fulliconthemepage)
        self.notebook.append_page(self.gtkthemes, self.gtkthemespage)
        self.notebook.append_page(self.gnomeshellthemes, self.gnomeshellthemespage)
        self.notebook.set_tab_reorderable(self.gridfulliconthemesscroll, True)
        #self.add(hb)
        
        #self.show_all()

        #threadtemp = threading.Thread(target=self.getImageFromWeb(each_item[0]['Image']))
        #self.threads.append(threadtemp)

        #self.getAllImages()
        x = threading.Thread(target=self.getAllImages(),daemon=True)
        x.start()
        self.show_all()

    def getProductCatalog(self):
        # Download Links from GnomeLook.org
        URL = "https://www.gnome-look.org/s/Gnome/browse/cat/132/page/2/ord/latest/"
        readgnomelook = ReadGnomeLook()
        self.productCatalog = readgnomelook.readWebpage(URL)
        #print(json.dumps(productCatalog, sort_keys=False, indent=4))

    def getAllImages(self):
        for i in range(0,len(self.productCatalog)):
            #self.images.append(self.getImageFromWeb(self.productCatalog[i][0]['Image']))
            #self.images[i] = self.getImageFromWeb(self.productCatalog[i][0]['Image'],self.images[i])
             with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
                 future = executor.submit(self.getImageFromWeb, self.productCatalog[i][0]['Image'], self.images[i])
            #     #self.images.append(future.result())
                 self.images[i]= future.result()
            #     #print(type(self.images[i]))

    def do_anything(self):
        print("clicked on box")

    def getImageFromWeb(self, URL,image):
        filename = '/tmp/'+URL.split("/")[-1]

        try:
            f = open(filename)
            pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(
                    filename=filename, 
                    width=100, 
                    height=100, 
                    preserve_aspect_ratio=False)

            Gtk.Image.set_from_pixbuf(image, pixbuf)
            #image.set_from_file(filename)
            #print("Got the image : " + filename)
            #del r
            return image
        except IOError:
            #print("File not accessible")
            r = requests.get(URL,stream=True)
            if r.status_code == 200:
                with open(filename,'wb') as f:
                    r.raw.decode_content = True
                    shutil.copyfileobj(r.raw, f)

                pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(
                        filename=filename, 
                        width=200, 
                        height=200, 
                        preserve_aspect_ratio=False)

                Gtk.Image.set_from_pixbuf(image, pixbuf)
                #image.set_from_file(filename)
                #print("Got the image : " + filename)
                return image
            else:
                #print("Failed to get the image : " + filename)
                return None
            del r


window = AppicationWindow()
#window.connect("destroy",Gtk.main_quit)
#window.show_all()
Gtk.main()

Code works fine. But in below code, the thread doProcessing() is getting completed and then I am seeing the "Started Z thread"

        print("Before 1st Show all")
        self.show_all()
        z = threading.Thread(target=self.doProcessing(),daemon=True)
        z.start()
        print("Started Z thread")

As I see, doProcessing should start in background and "Started Z thread" should be printed immediately but that's not happening.

Am I missing anything here ? Any help is appreciated.

Thanks, Debasish

Palmitoxico · 2021-08-17 12:44:20

It seems to be something related to python's Global Interpreter Lock. The default python interpreter (cpython) can not run more than one thread in parallel. That isn't a problem when dealing with I/O bound threads, as waiting for I/O will give CPU time to other thread, but it becomes a problem when dealing with CPU bound threads in multicore environments.

whitie · 2021-10-10 07:14:18

Maybe its to late for the reply, but the problem ist your Thread initialisation. You must give the function object as the target argument. You are actually calling the function.

NOT:

z = threading.Thread(target=self.doProcessing(),daemon=True)

BUT:

z = threading.Thread(target=self.doProcessing, daemon=True)

Whitie

Edit: You should never touch the GUI from another thread! It's broken by design.

Last edited by whitie (2021-10-10 07:18:55)

Arch Linux

#1 2021-05-30 22:13:43

Not getting the Multithreaading to work for GTK 3 python

#2 2021-08-17 12:44:20

Re: Not getting the Multithreaading to work for GTK 3 python

#3 2021-10-10 07:14:18

Re: Not getting the Multithreaading to work for GTK 3 python

Board footer