Comment

Comment by BioGeek

Using Biopython. Note that the search query that I am using currently returns 70605 results, so you might want to tweak it fit your needs.

    from Bio import Entrez
    import time
    from urllib.error import HTTPError
    
    DB = 'nucleotide'
    QUERY = '("pneumoviridae"[Organism] OR "Coronaviridae"[Organism])'
    
    Entrez.email = '[email protected]'
    handle = Entrez.esearch(db=DB, term=QUERY, rettype='fasta')
    record = Entrez.read(handle)
    
    handle = Entrez.esearch(db=DB, term=QUERY, retmax=count, rettype='fasta')
    record = Entrez.read(handle)

    id_list = record['IdList']
    count = len(id_list)
    post_xml = Entrez.epost(DB, id=",".join(id_list))
    search_results = Entrez.read(post_xml)
    
    webenv = search_results['WebEnv']
    query_key = search_results['QueryKey']

    batch_size = 200
    with open('viruses.fasta', 'w') as out_handle:
        for start in range(0, count, batch_size):
            end = min(count, start+batch_size)
            print(f"Going to download record {start+1} to {end}")
            attempt = 0
            success = False
            while attempt < 3 and not success:
                attempt += 1
                try:
                    fetch_handle = Entrez.efetch(db=DB, rettype='fasta',
                                                 retstart=start, retmax=batch_size,
                                                 webenv=webenv, query_key=query_key)
                    success = True
                except HTTPError as err:
                    if 500 <= err.code <= 599:
                        print(f"Received error from server {err}")
                        print("Attempt {attempt} of 3")
                        time.sleep(15)
                    else:
                        raise
            data = fetch_handle.read()
            fetch_handle.close()
            out_handle.write(data)
parent BioGeek 6 years ago

Discussion

Live from the official Hacker News API

A modern reader for Hacker News · SvelteKit + Skeleton