diff --git a/README.md b/README.md
index 7db82dc..0daff98 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
-# Botometer Python API
+[](https://badge.fury.io/py/botometer)
-A Python API for [Botometer by OSoMe](https://osome.iuni.iu.edu).
+# Botometer X Python API
+
+A Python API for [Botometer X by OSoMe](https://osome.iu.edu).
Previously known as `botornot-python`.
Behind the scenes, this uses the Botometer's HTTP endpoint, available via
@@ -8,163 +10,108 @@ Behind the scenes, this uses the Botometer's HTTP endpoint, available via
RapidAPI usage/account related questions should be posted on RapidAPI discussion.
-## [Change Note]
-### May, 2020
+## [Change Note/Announcement]
+
+### June, 2024
+
+We are releasing a new API endpoint for Botometer X.
-We have made some changes to our API, please read the [annoucnment](https://twitter.com/Botometer/status/1250557098708144131) for details. Due to the API change, the old `botometer-python` package might stop to work and produce 404 errors. Please upgrade it in your local environment to the least version.
+Unlike the original Botometer that fetched data from Twitter and calculated bot scores on the fly, Botometer X is in archival mode and relies on pre-calculated scores based on historical data collected before June 2023.
+The API endpoint allows users to fetch scores in bulk using a list of user ids or screen names, without the need of a Twitter/X's developer account.
-### Sep, 2019
+For details of Botometer X, please refer to the [FQA](https://botometer.osome.iu.edu/faq).
-Mashape has renamed itself to [RapidAPI](https://rapidapi.com/).
-The old mashape.com based URL and HTTP headers were deprecated in Sep 1st, 2019.
-So please upgrade `botometer-python` package in your local environment to the least version for the change.
## Help
-> You probably want to have a look at [Troubleshooting & FAQ](https://github.com/IUNetSci/botometer-python/wiki/Troubleshooting-&-FAQ) in the wiki. Please feel free to suggest and/or contribute improvements to that page.
+> You probably want to have a look at [Troubleshooting & FAQ](https://github.com/osome-iu/botometer-python/wiki/Troubleshooting-&-FAQ) in the wiki. Please feel free to suggest and/or contribute improvements to that page.
## Prior to Utilizing Botometer
-To begin using Botometer, you must follow the steps below before running any code:
+To begin using Botometer X, you must follow the steps below before running any code:
1. Create a free [RapidAPI](https://rapidapi.com/) account.
-2. Subscribe to [Botometer Pro](https://rapidapi.com/OSoMe/api/botometer-pro) on RapidApi.
+2. Subscribe to [Botometer Pro](https://rapidapi.com/OSoMe/api/botometer-pro) on RapidApi by selecting a plan.
> There is a completely free version (which does not require any credit card information) for testing purposes.
-3. Create a Twitter application via https://developer.twitter.com/
- > Botometer utilizes the access credentials provided by Twitter for the application.
-4. Ensure Botometer Pro's dependencies are already installed.
+3. Ensure Botometer Pro's dependencies are already installed.
> See the [Dependencies](#dependencies) section for details.
-**Note:** These steps are necessary to access credentials and download other packages which are needed for Botometer to work properly. Please see [RapidAPI and Twitter Access Details](#access) below for more details on this topic.
-
## Quickstart
-From your command shell, run
+From your command shell, run
```
pip install botometer
```
-then in a Python shell or script, enter something like this:
+### Botometer X
+
+To access the Botometer X endpoint, enter something like this in a Python shell or script:
+
```python
import botometer
-rapidapi_key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # now it's called rapidapi key
-twitter_app_auth = {
- 'consumer_key': 'xxxxxxxx',
- 'consumer_secret': 'xxxxxxxxxx',
- 'access_token': 'xxxxxxxxx',
- 'access_token_secret': 'xxxxxxxxxxx',
- }
-bom = botometer.Botometer(wait_on_ratelimit=True,
- rapidapi_key=rapidapi_key,
- **twitter_app_auth)
-
-# Check a single account by screen name
-result = bom.check_account('@clayadavis')
-
-# Check a single account by id
-result = bom.check_account(1548959833)
-
-# Check a sequence of accounts
-accounts = ['@clayadavis', '@onurvarol', '@jabawack']
-for screen_name, result in bom.check_accounts_in(accounts):
- # Do stuff with `screen_name` and `result`
+rapidapi_key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+bomx = botometer.BotometerX(rapidapi_key=rapidapi_key)
+```
+
+
+```python
+# Check accounts by usernames, note that @ is optional
+bomx.get_botscores_in_batch(usernames=['@OSoMe_IU', 'botometer'])
+
+# Check accounts by ids
+bomx.get_botscores_in_batch(user_ids=[2451308594, 187521608])
+
+# Check accounts by both usernames and ids
+bomx.get_botscores_in_batch(usernames=['@OSoMe_IU'], user_ids=[2451308594])
```
-Result:
+The queries will return results like those below:
+
```json
-{
- "cap": {
- "english": 0.0011785984309163565,
- "universal": 0.0016912294273666159
- },
- "categories": {
- "content": 0.058082395351262375,
- "friend": 0.044435259626385865,
- "network": 0.07064549990637549,
- "sentiment": 0.07214003430676995,
- "temporal": 0.07924665710801207,
- "user": 0.027817972609638725
- },
- "display_scores": {
- "content": 0.3,
- "english": 0.1,
- "friend": 0.2,
- "network": 0.4,
- "sentiment": 0.4,
- "temporal": 0.4,
- "universal": 0.1,
- "user": 0.1
- },
- "scores": {
- "english": 0.0215615093045025,
- "universal": 0.0254864249403189
- },
- "user": {
- "id_str": "1548959833",
- "screen_name": "clayadavis",
- "...": "..."
- }
-}
+[
+ {
+ "bot_score": 0.09,
+ "timestamp": "Sat, 27 May 2023 23:57:16 GMT",
+ "user_id": "2451308594",
+ "username": "Botometer"
+ },
+ {
+ "bot_score": 0.21,
+ "timestamp": "Thu, 25 May 2023 22:54:53 GMT",
+ "user_id": "187521608",
+ "username": "OSoMe_IU"
+ }
+]
```
+The response will be a list of JSON objects.
+Meanings of the elements in the object:
+- `bot_score`: The bot score, a float number between 0 and 1 (note that we rescale the score to 1 to 5 on the Botometer X website)
+- `timestamp`: The time when the bot score was calculated
+- `user_id`: ID of the account
+- `username`: Username of the account
+
+For more information on the API, consult the [API Overview](https://rapidapi.com/OSoMe/api/botometer-pro/details) on RapidAPI.
-For more information on this response object, consult the [API Overview](https://rapidapi.com/OSoMe/api/botometer-pro/details) on RapidAPI.
-## Install instructions
+## Installation instructions
+
This package is on PyPI so you can install it with pip:
```
$ pip install botometer
```
-
## Dependencies
### Python dependencies
* [requests](http://docs.python-requests.org/en/latest/)
-* [tweepy](https://github.com/tweepy/tweepy)
-
-Both of these dependencies are available via `pip`, so you can install both at once with
- pip install requests tweepy
-
-
-## RapidAPI and Twitter Access Details
-
-### RapidAPI key
-
-Our API is served via [RapidAPI](//rapidapi.com). You must sign up
-for a free account in order to obtain a RapidAPI secret key. The easiest way to
-get your secret key is to visit
-[our API endpoint page](https://rapidapi.com/OSoMe/api/botometer-pro/endpoints)
-and look in the endpoint's header parametsrs for the "X-RapidAPI-Key" as shown below:
-
-
-
-### Twitter app
-In order to access Twitter's API, one needs to have/create a [Twitter app](https://apps.twitter.com/).
-Once you've created an app, the authentication info can be found in the "Keys and Access Tokens" tab of the app's properties:
-
-
-## Authentication
-By default, Botometer uses **user authentication** when interacting with Twitter's API as it is the least restrictive and the ratelimit matches with Botometer's **Pro** plan: 180 requests per 15-minute window.
-One can instead use Twitter's **application authentication** in order to take advantage of the higher ratelimit that matches our **Ultra** plan: 450 requests per 15-minute window. Do note the differences between user and app-only authentication found under the header "Twitter API Authentication Model" in [Twitter's docs on authentication](https://developer.twitter.com/en/docs/basics/authentication/overview/oauth).
-
-To use app-only auth, just omit the `access_token` and `access_token_secret` in the `Botometer` constructor.
-
-```python
-import botometer
-
-rapidapi_key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # now it's called rapidapi key
-twitter_app_auth = {
- 'consumer_key': 'xxxxxxxx',
- 'consumer_secret': 'xxxxxxxxxx'
- }
-bom = botometer.Botometer(wait_on_ratelimit=True,
- rapidapi_key=rapidapi_key,
- **twitter_app_auth)
-```
+The dependency should be installed automatically with pip.
## References
-- ***Botometer v4:*** Mohsen Sayyadiharikandeh, Onur Varol, Kai-Cheng Yang, Alessandro Flammini, Filippo Menczer. "Detection of Novel Social Bots by Ensembles of Specialized Classifiers." [ArXiv](https://arxiv.org/abs/2006.06867)
+- **Botometer X:** The scores are calculated using the BotometerLite model.
+
+- ***Botometer v4:*** Mohsen Sayyadiharikandeh, Onur Varol, Kai-Cheng Yang, Alessandro Flammini, Filippo Menczer. "Detection of Novel Social Bots by Ensembles of Specialized Classifiers." [DOI](https://doi.org/10.1145/3340531.3412698), [ArXiv](https://arxiv.org/abs/2006.06867)
- ***BotometerLite:*** Yang, K.; Varol, O.; Hui, P.; and Menczer, F. "Scalable and Generalizable Social Bot Detection through Data Selection." AAAI (2020). [DOI](http://doi.org/10.1609/aaai.v34i01.5460), [ArXiv](https://arxiv.org/abs/1911.09179)
@@ -177,8 +124,3 @@ bom = botometer.Botometer(wait_on_ratelimit=True,
- Varol O., Davis C., Menczer, F., Flammini, A. "Feature Engineering for Social Bot Detection", Feature Engineering for Machine Learning and Data Analytics [Google Books](https://books.google.com/books?id=661SDwAAQBAJ&lpg=PA311&dq=info%3AsM983rg_yb8J%3Ascholar.google.com&lr&pg=PA311#v=onepage&q&f=false)
- Ferrara, Emilio, Onur Varol, Clayton Davis, Filippo Menczer, and Alessandro Flammini. "The rise of social bots." Communications of the ACM 59, no. 7 (2016): 96-104. [DOI](https://doi.org/10.1145/2818717), [ArXiv](https://arxiv.org/abs/1407.5225)
-
-
-```python
-
-```
diff --git a/botometer/__init__.py b/botometer/__init__.py
index 0d1c5c0..095589d 100644
--- a/botometer/__init__.py
+++ b/botometer/__init__.py
@@ -1,65 +1,20 @@
-from __future__ import print_function
-import time
-
import requests
-from requests import ConnectionError, HTTPError, Timeout
-import tweepy
-from tweepy.error import RateLimitError, TweepError
-
-
-class NoTimelineError(ValueError):
- def __init__(self, sn, *args, **kwargs):
- msg = "user '%s' has no tweets in timeline" % sn
- super(NoTimelineError, self).__init__(msg, *args, **kwargs)
-
-
-class Botometer(object):
- _TWITTER_RL_MSG = 'Rate limit exceeded for Twitter API method'
-
- def __init__(self,
- consumer_key, consumer_secret,
- access_token=None, access_token_secret=None,
- rapidapi_key=None,
- **kwargs):
- self.consumer_key = consumer_key
- self.consumer_secret = consumer_secret
- self.access_token_key = self.access_token = access_token
- self.access_token_secret = access_token_secret
- self.wait_on_ratelimit = kwargs.get('wait_on_ratelimit', False)
-
- self.rapidapi_key = rapidapi_key or kwargs.get('mashape_key')
- if self.access_token_key is None or self.access_token_secret is None:
- auth = tweepy.AppAuthHandler(
- self.consumer_key, self.consumer_secret)
- else:
- auth = tweepy.OAuthHandler(
- self.consumer_key, self.consumer_secret)
- auth.set_access_token(
- self.access_token_key, self.access_token_secret)
-
- self.twitter_api = tweepy.API(
- auth,
- parser=tweepy.parsers.JSONParser(),
- wait_on_rate_limit=self.wait_on_ratelimit,
- )
- self.api_url = kwargs.get('botometer_api_url',
- 'https://botometer-pro.p.rapidapi.com')
- self.api_version = kwargs.get('botometer_api_version', 2)
-
- @classmethod
- def create_from(cls, instance, **kwargs):
- my_kwargs = vars(instance)
- my_kwargs.update(kwargs)
- return cls(**my_kwargs)
+class BotometerBase(object):
+ def __init__(self, rapidapi_key, **kwargs):
+ self.rapidapi_key = rapidapi_key
+ self.api_url = kwargs.get(
+ "botometer_api_url", "https://botometer-pro.p.rapidapi.com"
+ )
def _add_rapidapi_header(self, kwargs):
if self.rapidapi_key:
- kwargs.setdefault('headers', {}).update({
- 'x-rapidapi-key': self.rapidapi_key
- })
+ kwargs.setdefault("headers", {}).update(
+ {"x-rapidapi-key": self.rapidapi_key}
+ )
+
return kwargs
def _bom_get(self, *args, **kwargs):
@@ -70,102 +25,76 @@ def _bom_post(self, *args, **kwargs):
self._add_rapidapi_header(kwargs)
return requests.post(*args, **kwargs)
- def _get_twitter_data(self, user, full_user_object=False):
- try:
- user_timeline = self.twitter_api.user_timeline(
- user,
- include_rts=True,
- count=200,
- )
+ ####################
+ ## Public methods ##
+ ####################
- except RateLimitError as e:
- e.args = (self._TWITTER_RL_MSG, 'statuses/user_timeline')
- raise e
+ def bom_api_path(self, method=""):
+ return "/".join(
+ [
+ self.api_url.rstrip("/"),
+ str(self.api_version),
+ method,
+ ]
+ )
- if user_timeline:
- user_data = user_timeline[0]['user']
- else:
- user_data = self.twitter_api.get_user(user)
- screen_name = '@' + user_data['screen_name']
- try:
- search = self.twitter_api.search(screen_name, count=100)
- except RateLimitError as e:
- e.args = (self._TWITTER_RL_MSG, 'search/tweets')
- raise e
+class BotometerX(BotometerBase):
+ """
+ Class to interact with the Botometer X API endpoint.
- payload = {
- 'mentions': search['statuses'],
- 'timeline': user_timeline,
- 'user': user_data,
- }
+ Lists of user_ids and/or screen_names with more than 100 elements would be truncated to 100.
+ Users are responsible to handle the exceptions.
+ """
- if not full_user_object:
- payload['user'] = {
- 'id_str': user_data['id_str'],
- 'screen_name': user_data['screen_name'],
- }
+ TWEETS_PER_REQUEST = 100
- return payload
+ def __init__(self, rapidapi_key, **kwargs):
+ super(BotometerX, self).__init__(rapidapi_key, **kwargs)
+ self.api_version = "botometer-x"
- ####################
- ## Public methods ##
- ####################
+ def _is_list_of_type(self, list_to_check, type_to_check):
+ if isinstance(list_to_check, list):
+ return all(isinstance(item, type_to_check) for item in list_to_check)
+ return False
- def bom_api_path(self, method=''):
- return '/'.join([
- self.api_url.rstrip('/'),
- str(self.api_version),
- method,
- ])
+ def get_botscores_in_batch(self, user_ids=None, usernames=None):
+ """
+ Get botscores based on a list of user_ids and/or screen_names.
+ There should be no more than 100 accounts in the query.
+ """
+ # Assign default values if not provided
+ user_ids = [] if user_ids is None else user_ids
+ usernames = [] if usernames is None else usernames
+ if not self._is_list_of_type(user_ids, int) and not self._is_list_of_type(
+ user_ids, str
+ ):
+ raise ValueError("user_ids must be a list of integers or strings")
- def check_account(self, user, full_user_object=False):
- payload = self._get_twitter_data(user,
- full_user_object=full_user_object)
- if not payload['timeline']:
- raise NoTimelineError(payload['user'])
+ if not self._is_list_of_type(usernames, str):
+ raise ValueError("usernames must be a list of strings")
+
+ if len(user_ids) == 0 and len(usernames) == 0:
+ raise ValueError("Must provide either user_ids or usernames")
+
+ N_BOTSCORES_PER_QUERY = 100
+
+ # Will only query the first N_BOTS_PER_QUERY items
+ if len(user_ids) > N_BOTSCORES_PER_QUERY:
+ # We have enough user ids, so we will query the first N_BOTS_PER_QUERY and ignore the usernames
+ user_ids = user_ids[:N_BOTSCORES_PER_QUERY]
+ usernames = []
+ else:
+ # We will query all the user ids plus N_BOTS_PER_QUERY - len(user_ids) usernames
+ usernames = usernames[: N_BOTSCORES_PER_QUERY - len(user_ids)]
+ payload = {
+ "user_ids": user_ids,
+ "usernames": usernames,
+ }
- url = self.bom_api_path('check_account')
+ url = self.bom_api_path("get_botscores_in_batch")
bom_resp = self._bom_post(url, json=payload)
bom_resp.raise_for_status()
- classification = bom_resp.json()
-
- return classification
-
-
- def check_accounts_in(self, accounts, full_user_object=False,
- on_error=None, **kwargs):
-
- sub_instance = self.create_from(self, wait_on_ratelimit=True,
- botometer_api_url=self.api_url)
- max_retries = kwargs.get('retries', 3)
-
- for account in accounts:
- for num_retries in range(max_retries + 1):
- result = None
- try:
- result = sub_instance.check_account(
- account, full_user_object=full_user_object)
- except (TweepError, NoTimelineError) as e:
- err_msg = '{}: {}'.format(
- type(e).__name__,
- getattr(e, 'msg', '') or getattr(e, 'reason', ''),
- )
- result = {'error': err_msg}
- except (ConnectionError, HTTPError, Timeout) as e:
- if num_retries >= max_retries:
- raise
- else:
- time.sleep(2 ** num_retries)
- except Exception as e:
- if num_retries >= max_retries:
- if on_error:
- on_error(account, e)
- else:
- raise
-
- if result is not None:
- yield account, result
- break
+ return bom_resp.json()
diff --git a/setup.py b/setup.py
index 0e1aecd..a49ab1b 100644
--- a/setup.py
+++ b/setup.py
@@ -1,16 +1,13 @@
from setuptools import setup
-setup(name='botometer',
- version='1.5',
- description='Check Twitter accounts for bot behavior',
- url='https://github.com/IUNetSci/botometer-python',
- download_url='https://github.com/IUNetSci/botometer-python/archive/1.0.zip',
- author='Clayton A Davis',
- author_email='claydavi@indiana.edu',
- license='MIT',
- packages=['botometer'],
- install_requires=[
- 'requests',
- 'tweepy >= 3.5.0',
- ],
- )
+setup(
+ name="botometer",
+ version="2.0.1",
+ description="Check Twitter accounts for bot behavior",
+ url="https://github.com/osome-iu/botometer-python",
+ author="Kai-Cheng Yang",
+ author_email="yang3kc@gmail.com",
+ license="MIT",
+ packages=["botometer"],
+ install_requires=["requests"],
+)