recipe for Calibre e-book management: can't make article bodies to appear in the output

Question

For Calibre, I am struggling to repair a built-in recipe for Vedomosti.

#!/usr/bin/env  python

u'''
Ведомости
'''

from calibre.web.feeds.feedparser import parse
from calibre.ebooks.BeautifulSoup import Tag
from calibre.web.feeds.news import BasicNewsRecipe

class VedomostiRecipe(BasicNewsRecipe):
    title = u'Ведомости'
    __author__ = 'Nikolai Kotchetkov'
    publisher = 'vedomosti.ru'
    category = 'press, Russia'
    description = u'Ежедневная деловая газета'
    oldest_article = 3
    max_articles_per_feed = 100

    masthead_url = u'http://motorro.com/imgdir/logos/ved_logo_black2_cropped.gif'
    cover_url = u'http://motorro.com/imgdir/logos/ved_logo_black2_cropped.gif'

    #Add feed names if you want them to be sorted (feeds of this list appear first)
    sortOrder = [u'_default', u'Первая полоса', u'Власть и деньги']

    encoding = 'cp1251'
    language = 'ru'
    no_stylesheets = True
    remove_javascript = True
    recursions = 0

    conversion_options = {
                          'comment'   : description
                        , 'tags'      : category
                        , 'publisher' : publisher
                        , 'language'  : language
                        }


    #keep_only_tags = [dict(name='td', attrs={'class' : ['second_content']})]

    #remove_tags_after = [dict(name='div', attrs={'class' : 'article_text'})]

    #remove_tags = [dict(name='div', attrs={'class' : ['sep', 'choice', 'articleRightTbl']})]

    feeds = [u'http://www.vedomosti.ru/newspaper/out/rss.xml']

    #base URL for relative links
    base_url = u'http://www.vedomosti.ru'

    extra_css = 'h1 {font-size: 1.5em; margin: 0em 0em 0em 0em; text-align: center;}'\
                'h2 {font-size: 1.0em; margin: 0em 0em 0em 0em;}'\
                'h3 {font-size: 0.8em; margin: 0em 0em 0em 0em;}'\
                '.article_date {font-size: 0.5em; color: gray; font-family: monospace; text-align:right;}'\
                '.article_authors {font-size: 0.5em; color: gray; font-family: monospace; text-align:right;}'\
                '.article_img {width:100%; text-align: center; padding: 3px 3px 3px 3px;}'\
                '.article_img_desc {width:100%; text-align: center; font-size: 0.5em; color: gray; font-family: monospace;}'\
                '.article_desc {font-size: 1em; font-style:italic;}'

    def parse_index(self):
        try:
            feedData = parse(self.feeds[0])
            if not feedData:
                raise NotImplementedError
            self.log("parse_index: Feed loaded successfully.")
            if feedData.feed.has_key('title'):
                self.title = feedData.feed.title
                self.log("parse_index: Title updated to: ", self.title)
            if feedData.feed.has_key('description'):
                self.description = feedData.feed.description
                self.log("parse_index: Description updated to: ", self.description)

            def get_virtual_feed_articles(feed):
                if feeds.has_key(feed):
                    return feeds[feed][1]
                self.log("Adding new feed: ", feed)
                articles = []
                feeds[feed] = (feed, articles)
                return articles

            feeds = {}

            #Iterate feed items and distribute articles using tags
            for item in feedData.entries:
                link = item.get('link', '');
                title = item.get('title', '');
                if '' == link or '' == title:
                    continue
                article = {'title':title, 'url':link, 'description':item.get('description', ''), 'date':item.get('date', ''), 'content':''};
                if not item.has_key('tags'):
                    get_virtual_feed_articles('_default').append(article)
                    continue
                for tag in item.tags:
                    addedToDefault = False
                    term = tag.get('term', '')
                    if '' == term:
                        if (not addedToDefault):
                            get_virtual_feed_articles('_default').append(article)
                        continue
                    get_virtual_feed_articles(term).append(article)

            #Get feed list
            #Select sorted feeds first of all
            result = []
            for feedName in self.sortOrder:
                if (not feeds.has_key(feedName)): continue
                result.append(feeds[feedName])
                del feeds[feedName]
            result = result + feeds.values()

            return result

        except Exception, err:
            self.log(err)
            raise NotImplementedError

    def preprocess_html(self, soup):
        return self.adeify_images(soup)

    def postprocess_html(self, soup, first_fetch):
        #self.log('Original: ', soup.prettify())

        #Find article
        contents = soup.find('div', {'class':['article_text']})
        if not contents:
            self.log('postprocess_html: article div not found!')
            return soup
        contents.extract()

        #Find title
        title = soup.find('h1')
        if title:
            contents.insert(0, title)

        #Find article image
        newstop = soup.find('div', {'class':['newstop']})
        if newstop:
            img = newstop.find('img')
            if img:
                imgDiv = Tag(soup, 'div')
                imgDiv['class'] = 'article_img'

                if img.has_key('width'):
                    del(img['width'])
                if img.has_key('height'):
                    del(img['height'])

                #find description
                element = img.parent.nextSibling

                img.extract()
                imgDiv.insert(0, img)

                while element:
                    if not isinstance(element, Tag):
                        continue
                    nextElement = element.nextSibling
                    if 'p' == element.name:
                        element.extract()
                        element['class'] = 'article_img_desc'
                        imgDiv.insert(len(imgDiv.contents), element)
                    element = nextElement

                contents.insert(1, imgDiv)

        #find article abstract
        abstract = soup.find('p', {'class':['subhead']})
        if abstract:
            abstract['class'] = 'article_desc'
            contents.insert(2, abstract)

        #Find article authors
        authorsDiv = soup.find('div', {'class':['autors']})
        if authorsDiv:
            authorsP = authorsDiv.find('p')
            if authorsP:
                authorsP['class'] = 'article_authors'
                contents.insert(len(contents.contents), authorsP)

        #Fix urls that use relative path
        urls = contents.findAll('a');
        if urls:
            for url in urls:
                if not url.has_key('href'):
                    continue
                if '/' == url['href'][0]:
                    url['href'] = self.base_url + url['href']

        body = soup.find('td', {'class':['second_content']})
        if body:
            body.replaceWith(contents)

        self.log('Result: ', soup.prettify())
        return soup

The problem I face is I can't make the article bodies to appear in the resulting .MOBI even with a most trivial reduction -- commenting out each of keep_only_tags, remove_tags_after, remove_tags variable settings. All I'm getting is tons of surrounding banners and navigation, but none of article texts. Log file doesn't have a single mention of "postprocess_html: article div not found" which looks like a primary signal of article text missing.

Of course, a trivial RSS-based custom news source for the same RSS works just fine: all article bodies get included.

Any suggestions on how to approach the problem?

Answer 1

Solved the problem with simply commenting out the whole body of postprocess_html method.