API Endpoint Reference

DEPRECATED!!! While we used to support API endpoints, we do not anymore as of April 2023.

Visual Genome API endpoint references allows you to easily access our database of images, graphs and region descriptions.

Method

Endpoint

Returns

Expand

GET

/api/v0/images/all

Return all image data (regions, graphs, QAs etc)

Name	Type	Description
[none]	int	ID of image


{
    "count": 108249,
    "next": "http://localhost:8000/api/v0/images/all?page=2",
    "previous": null,
    "results": [
        1,
        2,
        3,
        4,
        5,
        6,
        7,
        8,
        9,
        10,
        11,
        12,
        .
        .
        .
    ]
}

GET

/api/v0/images/{:id}

Return an image by ID

Name	Type	Description
id	int	ID of image
data_sets	str array	list of datasets in which image is included
url	hyperlink string	Visual Genome-hosted image URL
width	int	width of image in px
height	int	height of image in px


                      "id": 2412112,
                      "data_sets": [
                          "VG_100K"
                      ],
                      "url": "https://cs.stanford.edu/people/rak248/VG_100K_2/2412112.jpg",
                      "width": 391,
                      "height": 500

GET

/api/v0/images/{:id}/regions

Return all region descriptions for image

Name	Type	Description
image	int	ID of image containing region
x	int	x-coordinate of region bounding box
y	int	y-coordinate of region bounding box
width	int	width of region bounding box
height	int	height of region bounding box


[...
      {
          "image": 2407890,
          "x": 117,
          "y": 79,
          "width": 249,
          "height": 107,
          "phrase": "a cat sitting on a table.",
      },
      {
          "image": 2407890,
          "x": 116,
          "y": 29,
          "width": 239,
          "height": 135,
          "phrase": "a white cat with a tan tail and face markings",
      },
...]

GET

/api/v0/images/{:id}/regions/{:r_id}

Return all region graph for a region in an image

Name	Type	Description
image	int	ID of image containing region
x	int	x-coordinate of region bounding box
y	int	y-coordinate of region bounding box
width	int	width of region bounding box
height	int	height of region bounding box
bounding_boxes	object array	array of object bounding boxes contained in region
.id	int	ID of object bounding box
.x	int	x-coordinate of object bounding box
.y	int	y-coordinate of object bounding box
.width	int	width of object bounding box
.height	int	height of object bounding box
.boxed_objects	object array	names describing object in bounding box
.name	str	object name
.object_canon	str	object name
relationships	object array	array of relationships in region
.subject	int	ID of subject bounding box
.predicate	str	relationship predicate
.object	int	ID of object bounding box
.relationship_canon	object array	array containing canonicalized synsets
attributes	object array	array of attributes in region
.subject	int	ID of subject bounding box
.attribute	str	attribute
.attribute_canon	object array	array containing canonicalized synsets


[
      {
          "image": 2407890,
          "x": 116,
          "y": 29,
          "width": 239,
          "height": 135,
          "phrase": "a white cat with a tan tail and face markings",
          "bounding_boxes": [
              {
                  "id": 271872,
                  "x": 109,
                  "y": 37,
                  "width": 201,
                  "height": 133,
                  "boxed_objects": [
                      {
                          "name": "cat",
                          "object_canon": []
                      }
                  ]
              },
              {
                  "id": 271881,
                  "x": 112,
                  "y": 80,
                  "width": 75,
                  "height": 86,
                  "boxed_objects": [
                      {
                          "name": "tail",
                          "object_canon": []
                      }
                  ]
              },
              {
                  "id": 271882,
                  "x": 268,
                  "y": 61,
                  "width": 28,
                  "height": 31,
                  "boxed_objects": [
                      {
                          "name": "face",
                          "object_canon": []
                      }
                  ]
              },
              {
                  "id": 271883,
                  "x": 268,
                  "y": 64,
                  "width": 24,
                  "height": 28,
                  "boxed_objects": [
                      {
                          "name": "markings",
                          "object_canon": []
                      }
                  ]
              }
          ],
          "relationships": [
              {
                  "predicate": "has",
                  "subject": 271872,
                  "object": 271881,
                  "relationship_canon": [
                      {
                          "synset_name": "have.v.01",
                          "synset_definition": "have or possess, either in a concrete or an abstract sense"
                      }
                  ]
              },
              {
                  "predicate": "has",
                  "subject": 271872,
                  "object": 271882,
                  "relationship_canon": [
                      {
                          "synset_name": "have.v.01",
                          "synset_definition": "have or possess, either in a concrete or an abstract sense"
                      }
                  ]
              },
              {
                  "predicate": "has",
                  "subject": 271882,
                  "object": 271883,
                  "relationship_canon": [
                      {
                          "synset_name": "have.v.01",
                          "synset_definition": "have or possess, either in a concrete or an abstract sense"
                      }
                  ]
              }
          ],
          "attributes": [
              {
                  "attribute": "white",
                  "subject": 271872,
                  "attribute_canon": [
                      {
                          "synset_name": "white.a.01",
                          "synset_definition": "being of the achromatic color of maximum lightness; having little or no hue owing to reflection of almost all incident light"
                      }
                  ]
              },
              {
                  "attribute": "tan",
                  "subject": 271881,
                  "attribute_canon": [
                      {
                          "synset_name": "tan.s.01",
                          "synset_definition": "of a light yellowish-brown color"
                      }
                  ]
              }
          ]
      }
]

GET

/api/v0/images/{:id}/graph

Returns scene graph for specific image

Name	Type	Description
image	int	ID of image
bounding_boxes	object array	array of object bounding boxes contained in image
.id	int	ID of object bounding box
.x	int	x-coordinate of object bounding box
.y	int	y-coordinate of object bounding box
.width	int	width of object bounding box
.height	int	height of object bounding box
.boxed_objects	object array	names describing object in bounding box
.name	str	object name
.object_canon	str	object name
relationships	object array	array of relationships in image
.subject	int	ID of subject bounding box
.predicate	str	relationship predicate
.object	int	ID of object bounding box
.relationship_canon	object array	array containing canonicalized synsets
attributes	object array	array of attributes in image
.subject	int	ID of subject bounding box
.attribute	str	attribute
.attribute_canon	object array	array containing canonicalized synsets


{
  "image": 2414898,
  "bounding_boxes": [
      {
          "id": 149721,
          "x": 208,
          "y": 141,
          "width": 74,
          "height": 128,
          "boxed_objects": [
              {
                  "name": "woman",
                  "object_canon": []
              }
          ]
      },
      {
          "id": 149722,
          "x": 19,
          "y": 190,
          "width": 275,
          "height": 136,
          "boxed_objects": [
              {
                  "name": "bench",
                  "object_canon": []
              }
          ]
      },
      ...
  ],
  "relationships": [
      {
          "predicate": "sitting on",
          "subject": 149721,
          "object": 149722,
          "relationship_canon": [
              {
                  "synset_name": "sit.v.01",
                  "synset_definition": "be seated"
              }
          ]
      },
      {
          "predicate": "in",
          "subject": 149723,
          "object": 149723,
          "relationship_canon": []
      },
      {
          "predicate": "wearing",
          "subject": 149721,
          "object": 149724,
          "relationship_canon": [
              {
                  "synset_name": "wear.v.01",
                  "synset_definition": "be dressed in"
              }
          ]
      },
      ...
  ],
  "attributes": [
      {
          "attribute": "sitting",
          "subject": 149721,
          "attribute_canon": [
              {
                  "synset_name": "sit.v.01",
                  "synset_definition": "be seated"
              }
          ]
      },
      {
          "attribute": "pink",
          "subject": 149724,
          "attribute_canon": [
              {
                  "synset_name": "pink.s.01",
                  "synset_definition": "of a light shade of red"
              }
          ]
      },
...]
}

GET

/api/v0/image/{:id}/qa

Returns all QAs for specific image

Name	Type	Description
id	int	ID of QA pair
image	int	ID of image accessed
question	string	question sentence
answer	string	answer sentence
question_objects	object array	list of all identified objects in the question
.entity_start_idx	int	starting char index of entity
.entity_end_idx	int	ending char index of entity
.entity_name	int	name of recognized entity
.synset_name	int	unique synset name
.synset_definition	int	definition of synset according to WordNet
answer_objects	object array	list of all identified objects in the answer
.entity_start_idx	int	starting char index of entity
.entity_end_idx	int	ending char index of entity
.entity_name	int	name of recognized entity
.synset_name	int	unique synset name
.synset_definition	int	definition of synset according to WordNet


[...
      {
          "id": 2,
          "image": 1159826,
          "question": "Where is the man?",
          "answer": "In a bedroom.",
          "question_objects": [
              {
                  "entity_idx_start": 13,
                  "entity_idx_end": 16,
                  "entity_name": "man",
                  "synset_name": "man.n.01",
                  "synset_definition": "an adult person who is male (as opposed to a woman)"
              }
          ],
          "answer_objects": [
              {
                  "entity_idx_start": 5,
                  "entity_idx_end": 12,
                  "entity_name": "bedroom",
                  "synset_name": "bedroom.n.01",
                  "synset_definition": "a room used primarily for sleeping"
              }
          ]
      }
...]

GET

/api/v0/qa/all

Return all QAs in database

Name	Type	Description
id	int	ID of QA pair
image	int	ID of image accessed
question	string	question sentence
answer	string	answer sentence
question_objects	object array	list of all identified objects in the question
.entity_start_idx	int	starting char index of entity
.entity_end_idx	int	ending char index of entity
.entity_name	int	name of recognized entity
.synset_name	int	unique synset name
.synset_definition	int	definition of synset according to WordNet
answer_objects	object array	list of all identified objects in the answer
.entity_start_idx	int	starting char index of entity
.entity_end_idx	int	ending char index of entity
.entity_name	int	name of recognized entity
.synset_name	int	unique synset name
.synset_definition	int	definition of synset according to WordNet


[...
      {
          "id": 7,
          "image": 1159826,
          "question": "What is the object behind the man?",
          "answer": "A butterfly.",
          "question_objects": [
              {
                  "entity_idx_start": 12,
                  "entity_idx_end": 18,
                  "entity_name": "object",
                  "synset_name": "object.n.01",
                  "synset_definition": "a tangible and visible entity; an entity that can cast a shadow"
              },
              {
                  "entity_idx_start": 30,
                  "entity_idx_end": 33,
                  "entity_name": "man",
                  "synset_name": "man.n.01",
                  "synset_definition": "an adult person who is male (as opposed to a woman)"
              }
          ],
          "answer_objects": [
              {
                  "entity_idx_start": 2,
                  "entity_idx_end": 11,
                  "entity_name": "butterfly",
                  "synset_name": "butterfly.n.01",
                  "synset_definition": "diurnal insect typically having a slender body with knobbed antennae and broad colorful wings"
              }
          ]
      }
...]

GET

/api/v0/qa/:q_type

Return QA by type (how, when, where, why who, what)

Name	Type	Description
id	int	ID of QA pair
image	int	ID of image accessed
question	string	question sentence
answer	string	answer sentence
question_objects	object array	list of all identified objects in the question
.entity_start_idx	int	starting char index of entity
.entity_end_idx	int	ending char index of entity
.entity_name	int	name of recognized entity
.synset_name	int	unique synset name
.synset_definition	int	definition of synset according to WordNet
answer_objects	object array	list of all identified objects in the answer
.entity_start_idx	int	starting char index of entity
.entity_end_idx	int	ending char index of entity
.entity_name	int	name of recognized entity
.synset_name	int	unique synset name
.synset_definition	int	definition of synset according to WordNet


[...
      {
          "id": 45,
          "image": 1592710,
          "question": "Why do streets need traffic lights?",
          "answer": "To control traffic.",
          "question_objects": [
              {
                  "entity_idx_start": 20,
                  "entity_idx_end": 34,
                  "entity_name": "traffic lights",
                  "synset_name": "traffic_light.n.01",
                  "synset_definition": "a visual signal to control the flow of traffic at intersections"
              },
              {
                  "entity_idx_start": 7,
                  "entity_idx_end": 14,
                  "entity_name": "streets",
                  "synset_name": "street.n.01",
                  "synset_definition": "a thoroughfare (usually including sidewalks) that is lined with buildings"
              }
          ],
          "answer_objects": [
              {
                  "entity_idx_start": 11,
                  "entity_idx_end": 18,
                  "entity_name": "traffic",
                  "synset_name": "traffic.n.01",
                  "synset_definition": "the aggregation of things (pedestrians or vehicles) coming and going in a particular locality during a specified period of time"
              }
          ]
      }
...]