Commits

Lars Yencken committed 012b093

Longer language list from MultilingualBooks.

Comments (0)

Files changed (2)

mine_multilingual_books.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  mine_multilingual_books.py
+#  lang
+#
+
+"""
+Mine country's languages from MultilingualBooks.com.
+"""
+
+import codecs
+import sys
+
+import bs4
+import requests
+
+BASE_URL = 'http://www.multilingualbooks.com/fromcountry2lang.html'
+
+def mine_languages():
+    ostream = codecs.getwriter('utf8')(sys.stdout)
+    content = safe_get(BASE_URL).content
+    soup = bs4.BeautifulSoup(content)
+
+    table = soup.find('a', text=lambda t: t and 'Afghanistan' in
+            t).findParent('table')
+    for tr in table.findChildren('tr')[1:-1]:
+        country_td, lang_td = tr.findChildren('td')
+        try:
+            country = country_td.findChild('b').text
+        except AttributeError:
+            continue
+        languages = [l.text for l in lang_td.findChildren('a')]
+        for language in languages:
+            print >> ostream, '\t'.join((language, country))
+
+class ScrapingError(Exception):
+    pass
+
+def safe_get(*args, **kwargs):
+    response = requests.get(*args, **kwargs)
+    if response.status_code != 200:
+        raise ScrapingError('got status code %d' % response.status_code)
+    return response
+
+if __name__ == '__main__':
+    mine_languages()

mlbooks_country_language_2012-09-07.csv

+Afghan Persian or Dari	Afghanistan
+Pashtu	Afghanistan
+English	Akrotiri
+Greek	Akrotiri
+Albanian	Albania
+Greek	Albania
+Vlach	Albania
+Romani	Albania
+Slavic dialects	Albania
+Arabic	Algeria
+French	Algeria
+Berber dialects	Algeria
+Samoan	American Samoa
+English	American Samoa
+Tongan	American Samoa
+Catalan	Andorra
+French	Andorra
+Castilian Spanish	Andorra
+Portuguese	Andorra
+Portuguese	Angola
+Bantu	Angola
+English	Anguilla
+English	Antigua and Barbuda
+Spanish	Argentina
+English	Argentina
+Italian	Argentina
+German	Argentina
+French	Argentina
+Armenian	Armenia
+Yezidi	Armenia
+Russian	Armenia
+Dutch	Aruba
+Papiamento	Aruba
+English	Aruba
+Spanish	Aruba
+English	Australia
+Chinese	Australia
+Italian	Australia
+German	Austria
+Slovene	Austria
+Croatian	Austria
+Hungarian	Austria
+Azerbaijani	Azerbaijan
+Russian	Azerbaijan
+Armenian	Azerbaijan
+English	Bahamas
+Creole	Bahamas
+Arabic	Bahrain
+English	Bahrain
+Farsi	Bahrain
+Urdu	Bahrain
+Bengali	Bangladesh
+English	Bangladesh
+English	Barbados
+Belarusian	Belarus
+Russian	Belarus
+Dutch	Belgium
+French	Belgium
+German	Belgium
+English	Belize
+Spanish	Belize
+Mayan	Belize
+Garifuna	Belize
+Creole	Belize
+French	Benin
+Fon and Yoruba	Benin
+English	Bermuda
+Portuguese	Bermuda
+Dzongkha	Bhutan
+Tibetan	Bhutan
+Nepalese	Bhutan
+Spanish	Bolivia
+Quechua	Bolivia
+Aymara	Bolivia
+Bosnian	Bosnia and Herzegovina
+Croatian	Bosnia and Herzegovina
+Serbian	Bosnia and Herzegovina
+Setswana	Botswana
+Kalanga	Botswana
+Sekgalagadi	Botswana
+English	Botswana
+Portuguese	Brazil
+Spanish	Brazil
+English	Brazil
+French	Brazil
+English	British Virgin Islands
+Malay	Brunei
+English	Brunei
+Chinese	Brunei
+Bulgarian	Bulgaria
+Turkish	Bulgaria
+Roma	Bulgaria
+French	Burkina Faso
+Burmese	Burma
+Kirundi	Burundi
+French	Burundi
+Swahili	Burundi
+Khmer	Cambodia
+French	Cambodia
+English	Cambodia
+English	Cameroon
+French	Cameroon
+English	Canada
+French	Canada
+Portuguese	Cape Verde
+Crioulo	Cape Verde
+English	Cayman Islands
+French	Central African Republic
+Sangho	Central African Republic
+French	Chad
+Arabic	Chad
+Sara	Chad
+Spanish	Chile
+Mandarin	China
+Yue	China
+Wu	China
+Minbei	China
+Minnan	China
+Xiang	China
+Gan	China
+Hakka dialects	China
+English	Christmas Island
+Chinese	Christmas Island
+Malay	Christmas Island
+Malay	Cocos Islands
+English	Cocos Islands
+Spanish	Colombia
+Arabic	Comoros
+French	Comoros
+Shikomoro	Comoros
+French	Congo, Democratic Republic of the
+Lingala	Congo, Democratic Republic of the
+Kingwana	Congo, Democratic Republic of the
+Kikongo	Congo, Democratic Republic of the
+Tshiluba	Congo, Democratic Republic of the
+French	Congo, Republic of the
+Lingala and Monokutuba	Congo, Republic of the
+English	Cook Islands
+Maori	Cook Islands
+Spanish	Costa Rica
+English	Costa Rica
+French	Cote d'Ivoire
+Croatian	Croatia
+Serbian	Croatia
+Spanish	Cuba
+Greek	Cyprus
+Turkish	Cyprus
+English	Cyprus
+Czech	Czech Republic
+Danish	Denmark
+Faroese	Denmark
+Greenlandic	Denmark
+German	Denmark
+English	Dhekelia
+Greek	Dhekelia
+French	Djibouti
+Arabic	Djibouti
+Somali	Djibouti
+Afar	Djibouti
+English	Dominica
+French patois	Dominica
+Spanish	Dominican Republic
+Tetum	East Timor
+Portuguese	East Timor
+Indonesian	East Timor
+English	East Timor
+Galole	East Timor
+Mambae	East Timor
+Kemak	East Timor
+Spanish	Ecuador
+Arabic	Egypt
+Spanish	El Salvador
+Nahua	El Salvador
+Spanish	Equatorial Guinea
+French	Equatorial Guinea
+Fang	Equatorial Guinea
+Bubi	Equatorial Guinea
+Ibo	Equatorial Guinea
+Afar	Eritrea
+Arabic	Eritrea
+Tigre	Eritrea
+Kunama	Eritrea
+Tigrinya	Eritrea
+Estonian	Estonia
+Russian	Estonia
+Amharic	Ethiopia
+Tigrinya	Ethiopia
+Oromigna	Ethiopia
+Guaragigna	Ethiopia
+Somali	Ethiopia
+Arabic	Ethiopia
+English	Ethiopia
+English	Falkland Islands
+Faroese	Faroe Islands
+Danish	Faroe Islands
+English	Fiji
+Fijian	Fiji
+Hindustani	Fiji
+Finnish	Finland
+Swedish	Finland
+French	France
+Provencal	France
+Breton	France
+Alsatian	France
+Corsican	France
+Catalan	France
+Basque	France
+Flemish	France
+French	French Guiana
+French	French Polynesia
+Polynesian	French Polynesia
+French	Gabon
+Fang	Gabon
+Myene	Gabon
+Nzebi	Gabon
+Bapounou/Eschira	Gabon
+Bandjabi	Gabon
+English	Gambia
+Mandinka	Gambia
+Wolof	Gambia
+Fula	Gambia
+Arabic	Gaza Strip
+Hebrew	Gaza Strip
+English	Gaza Strip
+Georgian	Georgia
+Russian	Georgia
+Armenian	Georgia
+Azeri	Georgia
+Abkhaz	Georgia
+German	Germany
+English	Ghana
+Akan	Ghana
+Moshi-Dagomba	Ghana
+Ewe	Ghana
+Ga	Ghana
+English	Gibraltar
+Spanish	Gibraltar
+Italian	Gibraltar
+Portuguese	Gibraltar
+Greek	Greece
+English	Greece
+French	Greece
+Greenlandic	Greenland
+Danish	Greenland
+English	Greenland
+English	Grenada
+French patois	Grenada
+French	Guadeloupe
+Creole patois	Guadeloupe
+English	Guam
+Chamorro	Guam
+Spanish	Guatemala
+Quiche	Guatemala
+Cakchiquel	Guatemala
+Kekchi	Guatemala
+Mam	Guatemala
+Garifuna	Guatemala
+Xinca	Guatemala
+English	Guernsey
+French	Guernsey
+Norman-French	Guernsey
+French	Guinea
+Portuguese	Guinea-Bissau
+Crioulo	Guinea-Bissau
+English	Guyana
+Amerindian dialects	Guyana
+Creole	Guyana
+Hindi	Guyana
+Urdu	Guyana
+French	Haiti
+Creole	Haiti
+Italian	Holy See
+Latin	Holy See
+French	Holy See
+Spanish	Honduras
+Amerindian dialects	Honduras
+Chinese	Hong Kong
+English	Hong Kong
+Hungarian	Hungary
+Icelandic	Iceland
+English	Iceland
+Hindi	India
+English	India
+Bengali	India
+Telugu	India
+Marathi	India
+Tamil	India
+Urdu	India
+Gujarati	India
+Malayalam	India
+Kannada	India
+Oriya	India
+Punjabi	India
+Assamese	India
+Kashmiri	India
+Sindhi	India
+Sanskrit	India
+Hindustani	India
+Bahasa Indonesia	Indonesia
+English	Indonesia
+Dutch	Indonesia
+Javanese	Indonesia
+Persian and Persian dialects	Iran
+Turkic and Turkic dialects	Iran
+Kurdish	Iran
+Luri	Iran
+Balochi	Iran
+Arabic	Iran
+Turkish	Iran
+Arabic	Iraq
+Kurdish	Iraq
+Assyrian	Iraq
+Armenian	Iraq
+English	Ireland
+Irish Gaelic	Ireland
+English	Isle of Man
+Manx Gaelic	Isle of Man
+Hebrew	Israel
+Arabic	Israel
+Italian	Italy
+German	Italy
+French	Italy
+Slovene	Italy
+English	Jamaica
+English patois	Jamaica
+Japanese	Japan
+English	Jersey
+Portuguese	Jersey
+Arabic	Jordan
+Kazakh	Kazakhstan
+Russian	Kazakhstan
+English	Kenya
+Kiswahili	Kenya
+I-Kiribati	Kiribati
+English	Kiribati
+Korean	Korea, North
+Korean	Korea, South
+Arabic	Kuwait
+Kyrgyz	Kyrgyzstan
+Russian	Kyrgyzstan
+Lao	Laos
+French	Laos
+English	Laos
+Latvian	Latvia
+Russian	Latvia
+Lithuanian	Latvia
+Arabic	Lebanon
+French	Lebanon
+English	Lebanon
+Armenian	Lebanon
+Sesotho	Lesotho
+English	Lesotho
+Zulu	Lesotho
+Xhosa	Lesotho
+English	Liberia
+Arabic	Libya
+Italian	Libya
+English	Libya
+German	Liechtenstein
+Alemannic dialect	Liechtenstein
+Lithuanian	Lithuania
+Russian	Lithuania
+Polish	Lithuania
+Luxembourgish	Luxembourg
+German	Luxembourg
+French	Luxembourg
+Cantonese	Macau
+Hokkien	Macau
+Mandarin	Macau
+Macedonian	Macedonia
+Albanian	Macedonia
+Turkish	Macedonia
+Roma	Macedonia
+Serbian	Macedonia
+French	Madagascar
+Malagasy	Madagascar
+Chichewa	Malawi
+Chinyanja	Malawi
+Chiyao	Malawi
+Chitumbuka	Malawi
+Chisena	Malawi
+Chilomwe	Malawi
+Chitonga	Malawi
+Bahasa Melayu	Malaysia
+English	Malaysia
+Mandarin	Malaysia
+Cantonese	Malaysia
+Hokkien	Malaysia
+Hakka	Malaysia
+Hainan	Malaysia
+Foochow	Malaysia
+Tamil	Malaysia
+Telugu	Malaysia
+Malayalam	Malaysia
+Panjabi	Malaysia
+Thai	Malaysia
+Maldivian Dhivehi	Maldives
+English	Maldives
+French	Mali
+Bambara	Mali
+Maltese	Malta
+English	Malta
+Marshallese	Marshall Islands
+English	Marshall Islands
+French	Martinique
+Creole patois	Martinique
+Arabic	Mauritania
+Pulaar	Mauritania
+Soninke	Mauritania
+French	Mauritania
+Hassaniya	Mauritania
+Wolof	Mauritania
+Creole	Mauritius
+Bhojpuri	Mauritius
+French	Mauritius
+English	Mauritius
+Mahorian	Mayotte
+French	Mayotte
+Spanish	Mexico
+English	Micronesia, Federated States of
+Trukese	Micronesia, Federated States of
+Pohnpeian	Micronesia, Federated States of
+Yapese	Micronesia, Federated States of
+Kosrean	Micronesia, Federated States of
+Ulithian	Micronesia, Federated States of
+Woleaian	Micronesia, Federated States of
+Nukuoro	Micronesia, Federated States of
+Kapingamarangi	Micronesia, Federated States of
+Moldovan	Moldova
+Russian	Moldova
+Gagauz	Moldova
+French	Monaco
+English	Monaco
+Italian	Monaco
+Monegasque	Monaco
+Khalkha Mongol	Mongolia
+Turkic	Mongolia
+Russian	Mongolia
+Serbian	Montenegro
+English	Montserrat
+Arabic	Morocco
+Berber dialects	Morocco
+French	Morocco
+Emakhuwa	Mozambique
+Xichangana	Mozambique
+Portuguese	Mozambique
+Elomwe	Mozambique
+Cisena	Mozambique
+Echuwabo	Mozambique
+Nauruan	Nauru
+Nepali	Nepal
+Maithali	Nepal
+Bhojpuri	Nepal
+Tharu	Nepal
+Tamang	Nepal
+Newar	Nepal
+Magar	Nepal
+Awadhi	Nepal
+Dutch	Netherlands
+Frisian	Netherlands
+Papiamento	Netherlands Antilles
+English	Netherlands Antilles
+Dutch	Netherlands Antilles
+Spanish	Netherlands Antilles
+Creole	Netherlands Antilles
+French	New Caledonia
+33 Melanesian-Polynesian dialects	New Caledonia
+English	New Zealand
+Maori	New Zealand
+Spanish	Nicaragua
+Miskito	Nicaragua
+French	Niger
+Hausa	Niger
+Djerma	Niger
+English	Nigeria
+Hausa	Nigeria
+Yoruba	Nigeria
+Igbo	Nigeria
+Fulani	Nigeria
+Niuean	Niue
+English	Niue
+English	Norfolk Island
+Norfolk	Norfolk Island
+Chinese	Northern Mariana Islands
+Chamorro	Northern Mariana Islands
+English	Northern Mariana Islands
+Norwegian	Norway
+Bokmal Norwegian	Norway
+Nynorsk Norwegian	Norway
+Sami	Norway
+Finnish	Norway
+Arabic	Oman
+English	Oman
+Baluchi	Oman
+Urdu	Oman
+Indian dialects	Oman
+Punjabi	Pakistan
+Sindhi	Pakistan
+Siraiki	Pakistan
+Pashtu	Pakistan
+Urdu	Pakistan
+Balochi	Pakistan
+Hindko	Pakistan
+Brahui	Pakistan
+English	Pakistan
+Burushaski	Pakistan
+Palauan	Palau
+Angaur	Palau
+Filipino	Palau
+English	Palau
+Chinese	Palau
+Carolinian	Palau
+Japanese	Palau
+Spanish	Panama
+English	Panama
+Melanesian Pidgin	Papua New Guinea
+English	Papua New Guinea
+Motu	Papua New Guinea
+Spanish	Paraguay
+Guarani	Paraguay
+Spanish	Peru
+Quechua	Peru
+Aymara	Peru
+Tagalog	Philippines
+English	Philippines
+Cebuano	Philippines
+Ilocano	Philippines
+Hiligaynon or Ilonggo	Philippines
+Bicol	Philippines
+Waray	Philippines
+Pampango	Philippines
+Pangasinan	Philippines
+English	Pitcairn Islands
+Pitcairnese	Pitcairn Islands
+Polish	Poland
+Portuguese	Portugal
+Mirandese	Portugal
+Spanish	Puerto Rico
+English	Puerto Rico
+Arabic	Qatar
+French	Reunion
+Creole	Reunion
+Romanian	Romania
+Hungarian	Romania
+German	Romania
+Russian	Russia
+Kinyarwanda	Rwanda
+French	Rwanda
+English	Rwanda
+Kiswahili	Rwanda
+English	Saint Helena
+English	Saint Kitts and Nevis
+English	Saint Lucia
+French patois	Saint Lucia
+French	Saint Pierre and Miquelon
+English	Saint Vincent and the Grenadines
+French patois	Saint Vincent and the Grenadines
+Samoan	Samoa
+English	Samoa
+Italian	San Marino
+Portuguese	Sao Tome and Principe
+Arabic	Saudi Arabia
+French	Senegal
+Wolof	Senegal
+Pulaar	Senegal
+Jola	Senegal
+Mandinka	Senegal
+Serbian	Serbia
+Romanian	Serbia
+Hungarian	Serbia
+Slovak	Serbia
+Ukrainian	Serbia
+Croatian	Serbia
+Creole	Seychelles
+English	Seychelles
+English	Sierra Leone
+Mende	Sierra Leone
+Temne	Sierra Leone
+Krio	Sierra Leone
+Mandarin	Singapore
+English	Singapore
+Malay	Singapore
+Hokkien	Singapore
+Cantonese	Singapore
+Teochew	Singapore
+Tamil	Singapore
+Slovak	Slovakia
+Hungarian	Slovakia
+Roma	Slovakia
+Ukrainian	Slovakia
+Slovenian	Slovenia
+Serbo-Croatian	Slovenia
+Melanesian pidgin	Solomon Islands
+English	Solomon Islands
+Somali	Somalia
+Arabic	Somalia
+Italian	Somalia
+English	Somalia
+IsiZulu	South Africa
+IsiXhosa	South Africa
+Afrikaans	South Africa
+Sepedi	South Africa
+English	South Africa
+Setswana	South Africa
+Sesotho	South Africa
+Xitsonga	South Africa
+Castilian Spanish	Spain
+Catalan	Spain
+Galician	Spain
+Basque	Spain
+Sinhala	Sri Lanka
+Tamil	Sri Lanka
+Arabic	Sudan
+Nubian	Sudan
+Ta Bedawie	Sudan
+English	Sudan
+Dutch	Suriname
+English	Suriname
+Sranang Tongo (Surinamese)	Suriname
+Hindustani	Suriname
+Javanese	Suriname
+Norwegian	Svalbard
+Russian	Svalbard
+English	Swaziland
+siSwati	Swaziland
+Swedish	Sweden
+Sami	Sweden
+Finnish	Sweden
+German	Switzerland
+French	Switzerland
+Italian	Switzerland
+Serbo-Croatian	Switzerland
+Albanian	Switzerland
+Portuguese	Switzerland
+Spanish	Switzerland
+English	Switzerland
+Romansch	Switzerland
+Arabic	Syria
+Kurdish	Syria
+Armenian	Syria
+Aramaic	Syria
+Circassian	Syria
+French	Syria
+English	Syria
+Mandarin Chinese	Taiwan
+Taiwanese	Taiwan
+Hakka dialects	Taiwan
+Tajik	Tajikistan
+Russian	Tajikistan
+Kiswahili or Swahili	Tanzania
+Kiunguja	Tanzania
+English	Tanzania
+Arabic	Tanzania
+Thai	Thailand
+English	Thailand
+French	Togo
+Ewe and Mina	Togo
+Kabye	Togo
+Tokelauan	Tokelau
+English	Tokelau
+Tongan	Tonga
+English	Tonga
+English	Trinidad and Tobago
+Hindi	Trinidad and Tobago
+French	Trinidad and Tobago
+Spanish	Trinidad and Tobago
+Chinese	Trinidad and Tobago
+Arabic	Tunisia
+French	Tunisia
+Turkish	Turkey
+Kurdish	Turkey
+Dimli	Turkey
+Azeri	Turkey
+Kabardian	Turkey
+Gagauz	Turkey
+Turkmen	Turkmenistan
+Russian	Turkmenistan
+Uzbek	Turkmenistan
+English	Turks and Caicos Islands
+Tuvaluan	Tuvalu
+English	Tuvalu
+Samoan	Tuvalu
+Kiribati	Tuvalu
+English	Uganda
+Ganda or Luganda	Uganda
+Swahili	Uganda
+Arabic	Uganda
+Ukrainian	Ukraine
+Russian	Ukraine
+Romanian	Ukraine
+Polish	Ukraine
+Hungarian	Ukraine
+Arabic	United Arab Emirates
+Persian	United Arab Emirates
+English	United Arab Emirates
+Hindi	United Arab Emirates
+Urdu	United Arab Emirates
+English	United Kingdom
+Welsh	United Kingdom
+Scottish Gaelic	United Kingdom
+English	United States
+Spanish	United States
+Spanish	Uruguay
+Portunol	Uruguay
+Brazilero	Uruguay
+Uzbek	Uzbekistan
+Russian	Uzbekistan
+Tajik	Uzbekistan
+English	Vanuatu
+French	Vanuatu
+Spanish	Venezuela
+Vietnamese	Vietnam
+French	Vietnam
+English	Vietnam
+Chinese	Vietnam
+Khmer	Vietnam
+English	Virgin Islands
+Spanish	Virgin Islands
+Spanish Creole	Virgin Islands
+French	Virgin Islands
+French Creole	Virgin Islands
+Wallisian	Wallis and Futuna
+Futunian	Wallis and Futuna
+French	Wallis and Futuna
+Arabic	West Bank
+Hebrew	West Bank
+English	West Bank
+Hassaniya Arabic	Western Sahara
+Moroccan Arabic	Western Sahara
+Arabic	Yemen
+English	Zambia
+Bemba	Zambia
+Kaonda	Zambia
+Lozi	Zambia
+Lunda	Zambia
+Luvale	Zambia
+Nyanja	Zambia
+Tonga	Zambia
+English	Zimbabwe
+Shona	Zimbabwe
+Sindebele	Zimbabwe
+Ndebele	Zimbabwe