User-agent: heritrix Disallow: / User-agent: * Disallow: /good_neighbor_policy.html Disallow: /view/ViewUserProfile.do # Temporary until perf can be verified Disallow: /EditProfileBasic.do Disallow: /EditProfilePersonal.do Disallow: /EditProfileCareer.do Disallow: /EditProfilePrivacy.do Disallow: /EditProfileSettings.do Disallow: /FactValue.do Disallow: /FactHistoPopup.do Disallow: /FactNamesHistoPopup.do Disallow: /FactValue.do Disallow: /MyCommunity.do Disallow: /NewbieSetup.do Disallow: /PersonaFind.do Disallow: /SearchRequests.do Disallow: /ViewOthersFriends.do Disallow: /SubmitRequest.do Disallow: /SubmitRequestExp.do Disallow: /SubmitRequestExpa.do Disallow: /SubmitRequestExpb.do Disallow: /SubmitRequestExpc.do Disallow: /SubmitRequestExpd.do Disallow: /SubmitRequestExpe.do Disallow: /SubmitRequestPreview.do Disallow: /SignIn.do Disallow: /SignUpStart.do Disallow: /SignUpWithExistingAmazonAccount.do Disallow: /SignUpWithNewAmazonAccount.do Disallow: /AnswerBuilder.do Disallow: /AnswerBuilderResume.do Disallow: /AnswerBuilderAbandon.do Disallow: /AnswerBuilderSave.do Disallow: /AnswerBuilderSummary.do Disallow: /AnswerBuilderPreview.do Disallow: /AnswerBuilderWidgets.do Disallow: /AnswerBuilderStart.do Disallow: /AnswerBuilderDraft.do Disallow: /AnswerBuilderSegEdit.do Disallow: /AnswerBuilderSegPreview.do Disallow: /AnswerBuilderSegMove.do Disallow: /AnswerBuilderSegDelete.do Disallow: /keyword_question_howto.html Disallow: /AppealRating.do # Just to be safe, disallow community features relating to 1.7.1 Disallow: /MyActivities.do Disallow: /MyAccount.do Disallow: /MyWatchlist.do Disallow: /FriendsActivitySummary.do Disallow: /FavesActivitySummary.do Disallow: /MyHistory.do Disallow: /MyCompliments.do Disallow: /FriendActivities.do Disallow: /FaveActivities.do # disallow ajax calls Disallow: /ajax/* # consider not allowing indexing of any ajax Disallow: /ajax/PersonaRead.do Disallow: /ajax/SuggestFactValues.do Disallow: /ajax/SuggestFactValuesJson.do Disallow: /ajax/SuggestFactNames.do Disallow: /ajax/SuggestFactNamesJson.do Disallow: /ajax/AddValue.do Disallow: /ajax/GetMoreFactNames.do Disallow: /ajax/FactValue.do Disallow: /ajax/CommunityAction.do Disallow: /ajax/AskToBeFriend.do Disallow: /ajax/CompShop.do Allow: /ajax/RequestFinderByExample.do Allow: /ajax/TagQuestionFinder.do # just to be safe, this is for discuss this answer link Disallow: /*/DiscussionBoard.do?*&showPostForm=true Disallow: /*/AnswerDetails.do?*&showPostForm=true Disallow: /DiscussionBoard.do?*&showPostForm=true Disallow: /AnswerDetails.do?*&showPostForm=true Disallow: /*/DiscussionBoard.do?*&messageBody= Disallow: /DiscussionBoard.do?*&messageBody= Disallow: /*/DiscussionBoard.do?*scrollToLatestComment=true Disallow: /DiscussionBoard.do?*scrollToLatestComment=true Disallow: /JurorAction.do Disallow: /EmailContent.do Disallow: /EmailDigest.do Disallow: /Leaderboards.do