| CARVIEW |
Select Language
HTTP/2 200
etag: "sha256:d286c52a75c8ee9a5cf1fa89088a8d1c8160962c2dc451af21b9cb2ca7587a0a"
content-disposition: inline; filename="2501.08617v3.pdf"
last-modified: Wed, 11 Jun 2025 13:09:47 GMT
server: Google Frontend
x-cloud-trace-context: 89aab4bd6314a5b9ec7df39cc696ae68
via: 1.1 google, 1.1 varnish, 1.1 varnish, 1.1 varnish
access-control-allow-origin: *
content-type: application/pdf
link: ; rel='canonical'
accept-ranges: bytes
age: 19242
date: Thu, 01 Jan 2026 00:30:42 GMT
x-served-by: cache-lga21947-LGA, cache-lga21961-LGA, cache-bom-vanm7210068-BOM
x-cache: MISS, HIT, MISS
x-timer: S1767227442.224245,VS0,VE295
content-length: 2154127
%PDF-1.5
%????
1 0 obj
<< /Metadata 3 0 R /Names 4 0 R /OpenAction 5 0 R /Outlines 6 0 R /PageMode /UseOutlines /Pages 7 0 R /Type /Catalog >>
endobj
2 0 obj
<< /Author (Kaiqu Liang; Haimin Hu; Ryan Liu; Thomas L. Griffiths; Jaime Fern?ndez Fisac) /Creator (arXiv GenPDF \(tex2pdf:\)) /DOI (https://doi.org/10.48550/arXiv.2501.08617) /License (https://arxiv.org/licenses/nonexclusive-distrib/1.0/) /PTEX.Fullbanner (This is pdfTeX, Version 3.141592653-2.6-1.40.25 \(TeX Live 2023\) kpathsea version 6.3.5) /Producer (pikepdf 8.15.1) /Title (RLHS: Mitigating Misalignment in RLHF with Hindsight Simulation) /Trapped /False /arXivID (https://arxiv.org/abs/2501.08617v3) >>
endobj
3 0 obj
<< /Subtype /XML /Type /Metadata /Length 1732 >>
stream
RLHS: Mitigating Misalignment in RLHF with Hindsight Simulation Kaiqu Liang Haimin Hu Ryan Liu Thomas L. Griffiths Jaime Fernández Fisac https://arxiv.org/licenses/nonexclusive-distrib/1.0/ cs.LG cs.AI cs.CL
endstream
endobj
4 0 obj
<< /Dests 8 0 R >>
endobj
5 0 obj
<< /D [ 9 0 R /Fit ] /S /GoTo >>
endobj
6 0 obj
<< /Count 16 /First 10 0 R /Last 11 0 R /Type /Outlines >>
endobj
7 0 obj
<< /Count 27 /Kids [ 12 0 R 13 0 R 14 0 R 15 0 R 16 0 R ] /Type /Pages >>
endobj
8 0 obj
<< /Kids [ 17 0 R 18 0 R 19 0 R 20 0 R 21 0 R ] /Limits [ (Doc-Start) (theorem.1) ] >>
endobj
9 0 obj
<< /Annots [ 22 0 R 23 0 R 24 0 R 25 0 R 26 0 R 27 0 R 28 0 R 29 0 R 30 0 R 31 0 R 32 0 R 33 0 R 34 0 R 35 0 R 36 0 R 37 0 R 38 0 R 39 0 R 40 0 R 41 0 R 42 0 R 43 0 R 44 0 R 45 0 R 46 0 R 47 0 R 48 0 R 49 0 R ] /Contents [ 50 0 R 51 0 R ] /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources 52 0 R /Type /Page >>
endobj
10 0 obj
<< /A 53 0 R /Next 54 0 R /Parent 6 0 R /Title 55 0 R >>
endobj
11 0 obj
<< /A 56 0 R /Parent 6 0 R /Prev 57 0 R /Title 58 0 R >>
endobj
12 0 obj
<< /Count 6 /Kids [ 9 0 R 59 0 R 60 0 R 61 0 R 62 0 R 63 0 R ] /Parent 7 0 R /Type /Pages >>
endobj
13 0 obj
<< /Count 6 /Kids [ 64 0 R 65 0 R 66 0 R 67 0 R 68 0 R 69 0 R ] /Parent 7 0 R /Type /Pages >>
endobj
14 0 obj
<< /Count 6 /Kids [ 70 0 R 71 0 R 72 0 R 73 0 R 74 0 R 75 0 R ] /Parent 7 0 R /Type /Pages >>
endobj
15 0 obj
<< /Count 6 /Kids [ 76 0 R 77 0 R 78 0 R 79 0 R 80 0 R 81 0 R ] /Parent 7 0 R /Type /Pages >>
endobj
16 0 obj
<< /Count 3 /Kids [ 82 0 R 83 0 R 84 0 R ] /Parent 7 0 R /Type /Pages >>
endobj
17 0 obj
<< /Kids [ 85 0 R 86 0 R 87 0 R 88 0 R 89 0 R 90 0 R ] /Limits [ (Doc-Start) (cite.fernandes2023devil) ] >>
endobj
18 0 obj
<< /Kids [ 91 0 R 92 0 R 93 0 R 94 0 R 95 0 R 96 0 R ] /Limits [ (cite.fisher1970statistical) (cite.sun2024trustllm) ] >>
endobj
19 0 obj
<< /Kids [ 97 0 R 98 0 R 99 0 R 100 0 R 101 0 R 102 0 R ] /Limits [ (cite.taori2023stanford) (figure.caption.30) ] >>
endobj
20 0 obj
<< /Kids [ 103 0 R 104 0 R 105 0 R 106 0 R 107 0 R 108 0 R ] /Limits [ (figure.caption.31) (section*.20) ] >>
endobj
21 0 obj
<< /Kids [ 109 0 R 110 0 R 111 0 R 112 0 R 113 0 R 114 0 R ] /Limits [ (section*.21) (theorem.1) ] >>
endobj
22 0 obj
<< /A << /S /URI /Type /Action /URI (https://rl-hindsight.github.io) >> /Border [ 0 0 0 ] /C [ 0 1 1 ] /H /I /Rect [ 309.02 264.727 468.384 275.849 ] /Subtype /Link /Type /Annot >>
endobj
23 0 obj
<< /A << /D (cite.leike2018scalable) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 143.688 178.458 189.544 189.362 ] /Subtype /Link /Type /Annot >>
endobj
24 0 obj
<< /A << /D (cite.leike2018scalable) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 192.503 178.458 214.122 189.362 ] /Subtype /Link /Type /Annot >>
endobj
25 0 obj
<< /A << /D (cite.glaese2022improving) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 453.476 178.458 503.788 189.362 ] /Subtype /Link /Type /Annot >>
endobj
26 0 obj
<< /A << /D (cite.glaese2022improving) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 107.004 167.549 129.32 178.453 ] /Subtype /Link /Type /Annot >>
endobj
27 0 obj
<< /A << /D (cite.touvron2023llama) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 133.152 167.549 192.559 178.453 ] /Subtype /Link /Type /Annot >>
endobj
28 0 obj
<< /A << /D (cite.touvron2023llama) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 196.106 167.549 218.423 178.453 ] /Subtype /Link /Type /Annot >>
endobj
29 0 obj
<< /A << /D (cite.anthropic2023claude2) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 222.255 167.549 265.453 178.453 ] /Subtype /Link /Type /Annot >>
endobj
30 0 obj
<< /A << /D (cite.anthropic2023claude2) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 269.001 167.549 291.317 178.453 ] /Subtype /Link /Type /Annot >>
endobj
31 0 obj
<< /A << /D (cite.achiam2023gpt) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 295.149 167.549 352.528 178.453 ] /Subtype /Link /Type /Annot >>
endobj
32 0 obj
<< /A << /D (cite.achiam2023gpt) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 356.075 167.549 378.392 178.453 ] /Subtype /Link /Type /Annot >>
endobj
33 0 obj
<< /A << /D (cite.christiano2017deep) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 276.162 156.64 340.955 167.543 ] /Subtype /Link /Type /Annot >>
endobj
34 0 obj
<< /A << /D (cite.christiano2017deep) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 343.926 156.64 365.644 167.543 ] /Subtype /Link /Type /Annot >>
endobj
35 0 obj
<< /A << /D (cite.ziegler2019fine) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 368.891 156.64 420.92 167.543 ] /Subtype /Link /Type /Annot >>
endobj
36 0 obj
<< /A << /D (cite.ziegler2019fine) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 423.89 156.64 445.609 167.543 ] /Subtype /Link /Type /Annot >>
endobj
37 0 obj
<< /A << /D (cite.ouyang2022training) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 448.855 156.64 503.776 167.543 ] /Subtype /Link /Type /Annot >>
endobj
38 0 obj
<< /A << /D (cite.ouyang2022training) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 107.004 145.73 128.523 156.634 ] /Subtype /Link /Type /Annot >>
endobj
39 0 obj
<< /A << /D (cite.stiennon2020learning) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 131.206 145.73 188.378 156.634 ] /Subtype /Link /Type /Annot >>
endobj
40 0 obj
<< /A << /D (cite.stiennon2020learning) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 190.788 145.73 212.307 156.634 ] /Subtype /Link /Type /Annot >>
endobj
41 0 obj
<< /A << /D (cite.casper2023open) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 247.142 102.094 298.607 112.998 ] /Subtype /Link /Type /Annot >>
endobj
42 0 obj
<< /A << /D (cite.casper2023open) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 301.59 102.094 323.488 112.998 ] /Subtype /Link /Type /Annot >>
endobj
43 0 obj
<< /A << /D (cite.pandey2022modeling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 326.75 102.094 379.578 112.998 ] /Subtype /Link /Type /Annot >>
endobj
44 0 obj
<< /A << /D (cite.pandey2022modeling) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 382.562 102.094 404.459 112.998 ] /Subtype /Link /Type /Annot >>
endobj
45 0 obj
<< /A << /D (cite.chmielewski2020mturk) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 407.721 102.094 503.753 112.998 ] /Subtype /Link /Type /Annot >>
endobj
46 0 obj
<< /A << /D (cite.chmielewski2020mturk) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 107.004 91.185 128.523 102.089 ] /Subtype /Link /Type /Annot >>
endobj
47 0 obj
<< /A << /D (cite.lang2024your) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 427.768 80.276 472.091 91.18 ] /Subtype /Link /Type /Annot >>
endobj
48 0 obj
<< /A << /D (cite.lang2024your) /S /GoTo >> /Border [ 0 0 0 ] /C [ 0 1 0 ] /H /I /Rect [ 475.08 80.276 496.998 91.18 ] /Subtype /Link /Type /Annot >>
endobj
49 0 obj
<< /A << /S /URI /URI (https://arxiv.org/abs/2501.08617v3) >> /BS << /W 0 >> /NM (fitz-L0) /Rect [ 12 223.23999 32 568.76 ] /Subtype /Link >>
endobj
50 0 obj
<< /Filter /FlateDecode /Length 139 >>
stream
x?EI;
1??s??ɼL??X"ع???F????̪ ???ЃV?,;6??c?ot,?N<?J?t??e?{?7??̝?8l??y??"?(ڇy?(,???=b?!V?"???G=MK{??5???tyH[Z'???1'
endstream
endobj
51 0 obj
<< /Filter /FlateDecode /Length 3093 >>
stream
xڵZYs?6~????F]e+?$?Ⱦ???+?T&3N?j3yPK?wttt?????Zj?'')?H?x?ć ???;?yq%Ξ?\Ix
G:R$N,/????+i?^?(???????/?>????=_?й?9R / R'?/U?sW8??o_?|????W???f?
??O?>?̾?u3??4?A?7??}??؋?Թ?????f?7CI?^???;???L=V0??ثgwg[??ԋ???~N?v?????m??nS8??n?CX:?#??0?|?????&3????k???`m?,???ГqOXM?4?Mg?\-o???l|??????]o?#M???'}???C凱??a?????J??W?&??%?5??/3SOg?r??9?^??/??_MT??˥??^?>R??-?{̚??/*??????^??S????% <???+?:?Y??/:?{/??????H/??/؍?Po5?????GP{!L????Q??qyݡ?y??)?o??=????S??7y?w??j_`???`L??O??B?0O????m?tY>X:9?˳}????5a?%Q?Q??0ӏ??????}?M?k?\3??κ?hZ??????X[???s??m???{?r$<%IJ%x??O??`2?kK?????*??T?????\K?V????U7~?6?^?????=???P???L?U?M[?B?!H2??@o???ټ~?N??#??>?O???1F?5"?w?c?t
??(?n??9?t?/?7q?j?\n?{???(?ϐ??tn?f-?/?Ǎ?/?? ?@???`)?h#w{ħ?95 n_}?Q??S??0?? ?.Ɯ??h̺ۢaٽ????(?MV??G4??OhX '?&Յ]?^?Xwа??@?G|dMV{?h?ܓ????[0'?b??Z??a?h???IW??-t?~?/GSsj????oR?\?(?$???:օCR?i??k???%<.??}?T??????lU???®ۄʥ3?ExVP<???a?<??c?;???t?A?<?q_?+?&??UA4[?A@?'?? E?fF??I0??@?胧?j8?KW&7\?As?? ? 4?8?L??iS?Z(???oY}Oҡ?4?CWȚ1{??P?Gu?}@?????(@?Ɣ??t?X.X?5p?C6v???<?b?YS?KY}??[?W?^s??WӃn???3?w?R5?????;Kk?1??H_h?$t[Ks???4]????ӱ@???FFr??Ze[????v@<??????GHr???'?74
?I ?vr?????????+?5?}???Q???;S3?AH??K?ɯ??2Z
YGĀ?DZB/?p]s?퇛???],?-d??P???ntZ?D?j??n(??r7V??^??%t~?JZQ?{??w?Ruc??~??5???0C5&???R(xD{?Yd?n?gl#\?? Y??+???2?_??HX`??E?????t?ɟ?o???????x?f@vF??X[?f?F_?=? Ϋ?u??????Rt89걌??j???f??? w?????zE?(??M&(?lٞ??A{-&Q7??B?SeDʘNU"?y?????-֊?buJ !nzB?iUá???ϻ?fs???VI?ۛ???iW%?Rt?xSP??.U?˛2?O?;?|@9Rᾂ`?????#1Q???^b?x??!?v.-?%!-7?PL?H??^A??U??n_1?QM7?^?|??l?Qh?43???ʒ??'?j?????????!/2s%'??)?????QHI?06K??K??[?? ,Q? {??????8?l
??B?r\?>[?h??w
?? ?hA,Tl????x3r?.|g??????I>???ꧢ??N Wp?Z
????*??V???G'????H·Q"E??v?????y??L?X2????x